From b336ab42dcc81a351b2f875f28c70b74d8814611 Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Thu, 4 Jan 2024 16:40:13 +0100 Subject: [PATCH 001/728] [mlir] add a way to query non-property attributes (#76959) This helps support generic manipulation of operations that don't (yet) use properties to store inherent attributes. Use this mechanism in type inference and operation equivalence. Note that only minimal unit tests are introduced as all the upstream dialects seem to have been updated to use properties and the non-property behavior is essentially deprecated and untested. --- mlir/include/mlir/IR/Operation.h | 3 +++ mlir/lib/IR/OperationSupport.cpp | 9 ++++----- mlir/lib/Interfaces/InferTypeOpInterface.cpp | 5 ++--- mlir/unittests/Bytecode/BytecodeTest.cpp | 3 +++ mlir/unittests/IR/OpPropertiesTest.cpp | 10 ++++++++++ 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h index d2f52cf1afee..3ffd3517fe5a 100644 --- a/mlir/include/mlir/IR/Operation.h +++ b/mlir/include/mlir/IR/Operation.h @@ -500,6 +500,9 @@ public: llvm::to_vector(getDiscardableAttrs())); } + /// Return all attributes that are not stored as properties. + DictionaryAttr getRawDictionaryAttrs() { return attrs; } + /// Return all of the attributes on this operation. ArrayRef getAttrs() { return getAttrDictionary().getValue(); } diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp index eaad6f289160..e10cd748e03b 100644 --- a/mlir/lib/IR/OperationSupport.cpp +++ b/mlir/lib/IR/OperationSupport.cpp @@ -675,7 +675,7 @@ llvm::hash_code OperationEquivalence::computeHash( // - Attributes // - Result Types llvm::hash_code hash = - llvm::hash_combine(op->getName(), op->getDiscardableAttrDictionary(), + llvm::hash_combine(op->getName(), op->getRawDictionaryAttrs(), op->getResultTypes(), op->hashProperties()); // - Location if required @@ -831,14 +831,13 @@ OperationEquivalence::isRegionEquivalentTo(Region *lhs, Region *rhs, // 1. Compare the operation properties. if (lhs->getName() != rhs->getName() || - lhs->getDiscardableAttrDictionary() != - rhs->getDiscardableAttrDictionary() || + lhs->getRawDictionaryAttrs() != rhs->getRawDictionaryAttrs() || lhs->getNumRegions() != rhs->getNumRegions() || lhs->getNumSuccessors() != rhs->getNumSuccessors() || lhs->getNumOperands() != rhs->getNumOperands() || lhs->getNumResults() != rhs->getNumResults() || !lhs->getName().compareOpProperties(lhs->getPropertiesStorage(), - rhs->getPropertiesStorage())) + rhs->getPropertiesStorage())) return false; if (!(flags & IgnoreLocations) && lhs->getLoc() != rhs->getLoc()) return false; @@ -923,7 +922,7 @@ OperationFingerPrint::OperationFingerPrint(Operation *topOp) { if (op != topOp) addDataToHash(hasher, op->getParentOp()); // - Attributes - addDataToHash(hasher, op->getDiscardableAttrDictionary()); + addDataToHash(hasher, op->getRawDictionaryAttrs()); // - Properties addDataToHash(hasher, op->hashProperties()); // - Blocks in Regions diff --git a/mlir/lib/Interfaces/InferTypeOpInterface.cpp b/mlir/lib/Interfaces/InferTypeOpInterface.cpp index ee4c0519b9f5..e52d0e17cda2 100644 --- a/mlir/lib/Interfaces/InferTypeOpInterface.cpp +++ b/mlir/lib/Interfaces/InferTypeOpInterface.cpp @@ -240,9 +240,8 @@ LogicalResult mlir::detail::verifyInferredResultTypes(Operation *op) { auto retTypeFn = cast(op); auto result = retTypeFn.refineReturnTypes( op->getContext(), op->getLoc(), op->getOperands(), - op->getPropertiesStorage() ? op->getDiscardableAttrDictionary() - : op->getAttrDictionary(), - op->getPropertiesStorage(), op->getRegions(), inferredReturnTypes); + op->getRawDictionaryAttrs(), op->getPropertiesStorage(), op->getRegions(), + inferredReturnTypes); if (failed(result)) op->emitOpError() << "failed to infer returned types"; diff --git a/mlir/unittests/Bytecode/BytecodeTest.cpp b/mlir/unittests/Bytecode/BytecodeTest.cpp index 76ff1a8194db..bb7241c2d519 100644 --- a/mlir/unittests/Bytecode/BytecodeTest.cpp +++ b/mlir/unittests/Bytecode/BytecodeTest.cpp @@ -144,4 +144,7 @@ TEST(Bytecode, OpWithoutProperties) { EXPECT_EQ(roundtripped->getAttrs().size(), 2u); EXPECT_TRUE(roundtripped->getInherentAttr("inherent_attr") != std::nullopt); EXPECT_TRUE(roundtripped->getDiscardableAttr("other_attr") != Attribute()); + + EXPECT_TRUE(OperationEquivalence::computeHash(op.get()) == + OperationEquivalence::computeHash(roundtripped)); } diff --git a/mlir/unittests/IR/OpPropertiesTest.cpp b/mlir/unittests/IR/OpPropertiesTest.cpp index bb1b741d1cc2..365775d541ec 100644 --- a/mlir/unittests/IR/OpPropertiesTest.cpp +++ b/mlir/unittests/IR/OpPropertiesTest.cpp @@ -8,6 +8,7 @@ #include "mlir/IR/Attributes.h" #include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OperationSupport.h" #include "mlir/Parser/Parser.h" #include "gtest/gtest.h" #include @@ -401,6 +402,15 @@ TEST(OpPropertiesTest, withoutPropertiesDiscardableAttrs) { op->print(os); EXPECT_TRUE(StringRef(os.str()).contains("inherent_attr = 42")); EXPECT_TRUE(StringRef(os.str()).contains("other_attr = 56")); + + OwningOpRef reparsed = parseSourceString(os.str(), config); + auto trivialHash = [](Value v) { return hash_value(v); }; + auto hash = [&](Operation *operation) { + return OperationEquivalence::computeHash( + operation, trivialHash, trivialHash, + OperationEquivalence::Flags::IgnoreLocations); + }; + EXPECT_TRUE(hash(op.get()) == hash(reparsed.get())); } } // namespace -- GitLab From a87fa7f0caa1b8bf328524e0aec80a10ed4af7f9 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 4 Jan 2024 13:04:31 -0300 Subject: [PATCH 002/728] [InstCombine] Dont throw away noalias/alias scope metadata when inlining memcpys (#74805) This was found in julia when we changed some operations from explicit loads + stores to memcpys. While applying it to both the src and the dest seems weird, thats what we do for normal TBAA. --- .../Transforms/InstCombine/InstCombineCalls.cpp | 16 +++++++--------- .../Transforms/InstCombine/memcpy-to-load.ll | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 3da9b89a6409..40b48699f758 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -172,10 +172,10 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) { // If the memcpy has metadata describing the members, see if we can get the // TBAA tag describing our copy. - MDNode *CopyMD = nullptr; - if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa)) { - CopyMD = M; - } else if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa_struct)) { + AAMDNodes AACopyMD = MI->getAAMetadata(); + + if (MDNode *M = AACopyMD.TBAAStruct) { + AACopyMD.TBAAStruct = nullptr; if (M->getNumOperands() == 3 && M->getOperand(0) && mdconst::hasa(M->getOperand(0)) && mdconst::extract(M->getOperand(0))->isZero() && @@ -184,7 +184,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) { mdconst::extract(M->getOperand(1))->getValue() == Size && M->getOperand(2) && isa(M->getOperand(2))) - CopyMD = cast(M->getOperand(2)); + AACopyMD.TBAA = cast(M->getOperand(2)); } Value *Src = MI->getArgOperand(1); @@ -192,8 +192,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) { LoadInst *L = Builder.CreateLoad(IntType, Src); // Alignment from the mem intrinsic will be better, so use it. L->setAlignment(*CopySrcAlign); - if (CopyMD) - L->setMetadata(LLVMContext::MD_tbaa, CopyMD); + L->setAAMetadata(AACopyMD); MDNode *LoopMemParallelMD = MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access); if (LoopMemParallelMD) @@ -205,8 +204,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) { StoreInst *S = Builder.CreateStore(L, Dest); // Alignment from the mem intrinsic will be better, so use it. S->setAlignment(*CopyDstAlign); - if (CopyMD) - S->setMetadata(LLVMContext::MD_tbaa, CopyMD); + S->setAAMetadata(AACopyMD); if (LoopMemParallelMD) S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD); if (AccessGroupMD) diff --git a/llvm/test/Transforms/InstCombine/memcpy-to-load.ll b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll index 2f13394b5e47..f8e8ab85f935 100644 --- a/llvm/test/Transforms/InstCombine/memcpy-to-load.ll +++ b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll @@ -79,3 +79,19 @@ define void @copy_16_bytes(ptr %d, ptr %s) { ret void } +define void @copy_8_bytes_noalias(ptr %d, ptr %s) { +; CHECK-LABEL: @copy_8_bytes_noalias( +; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[S:%.*]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]] +; CHECK-NEXT: store i64 [[TMP1]], ptr [[D:%.*]], align 1, !alias.scope [[META0]], !noalias [[META3]] +; CHECK-NEXT: ret void +; + call void @llvm.memcpy.p0.p0.i32(ptr %d, ptr %s, i32 8, i1 false), !alias.scope !4, !noalias !5 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1} +!2 = !{!2, !0} +!3 = !{!3, !1} +!4 = !{!2} +!5 = !{!3} -- GitLab From 9215741726e295d09ae7db4d235b26c1214a19ae Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Thu, 4 Jan 2024 11:08:36 -0500 Subject: [PATCH 003/728] [mlir] Make fold result type check more verbose (#76867) Print the op and its types when the fold type check fails. This is to speed up debuging as it should be trivial to map the offending op to its folder based on the op name. --- mlir/lib/IR/Operation.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp index a726790391a0..311f5bb5ef77 100644 --- a/mlir/lib/IR/Operation.cpp +++ b/mlir/lib/IR/Operation.cpp @@ -20,6 +20,7 @@ #include "mlir/Interfaces/FoldInterfaces.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/Support/ErrorHandling.h" #include #include @@ -611,11 +612,19 @@ void Operation::setSuccessor(Block *block, unsigned index) { /// the results of the given op. static void checkFoldResultTypes(Operation *op, SmallVectorImpl &results) { - if (!results.empty()) - for (auto [ofr, opResult] : llvm::zip_equal(results, op->getResults())) - if (auto value = ofr.dyn_cast()) - assert(value.getType() == opResult.getType() && - "folder produced value of incorrect type"); + if (results.empty()) + return; + + for (auto [ofr, opResult] : llvm::zip_equal(results, op->getResults())) { + if (auto value = dyn_cast(ofr)) { + if (value.getType() != opResult.getType()) { + op->emitOpError() << "folder produced a value of incorrect type: " + << opResult.getType() + << ", expected: " << value.getType(); + assert(false && "incorrect fold result type"); + } + } + } } #endif // NDEBUG -- GitLab From 640ef55bbbc081b72a87f71cab1bce08762e48b0 Mon Sep 17 00:00:00 2001 From: Krystian Stasiowski Date: Thu, 4 Jan 2024 11:30:48 -0500 Subject: [PATCH 004/728] Reapply "[Clang][Sema] Diagnose unexpanded packs in the template argument lists of function template specializations" (#76876) (#76915) This reapplies f034044ad94d6f7ccec13d89f08acac257ed28bb after it was reverted by 687396b5f4ba0713d103ebd172b308e92eb930cc due to a test failure in clang-doc. The test in question declares a partial specialization of a function template, as well as an explicit specialization of the same function template. Both declarations are now set as invalid, meaning neither is emitted by clang-doc. Since this is the sole test of function template specializations in clang-doc, I presume the intent is for the partial specialization to actually be the primary template. Doing so results in the expected output. --- .../test/clang-doc/templates.cpp | 2 +- clang/docs/ReleaseNotes.rst | 1 + clang/lib/Sema/SemaDecl.cpp | 89 +++++++++---------- .../CXX/temp/temp.decls/temp.variadic/p5.cpp | 12 +++ 4 files changed, 57 insertions(+), 47 deletions(-) diff --git a/clang-tools-extra/test/clang-doc/templates.cpp b/clang-tools-extra/test/clang-doc/templates.cpp index eb7f4599629f..2e04a77ac9e6 100644 --- a/clang-tools-extra/test/clang-doc/templates.cpp +++ b/clang-tools-extra/test/clang-doc/templates.cpp @@ -7,7 +7,7 @@ // RUN: rm -rf %t template -void function(T x) {} +void function(T x) {} template<> void function(bool x) {} diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 30cfe66703f5..c7bf162426a6 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -518,6 +518,7 @@ Improvements to Clang's diagnostics - Clang now diagnoses definitions of friend function specializations, e.g. ``friend void f<>(int) {}``. - Clang now diagnoses narrowing conversions involving const references. (`#63151: `_). +- Clang now diagnoses unexpanded packs within the template argument lists of function template specializations. Improvements to Clang's time-trace diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 2de631941325..8e46c4984d93 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -9900,15 +9900,15 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, // Match up the template parameter lists with the scope specifier, then // determine whether we have a template or a template specialization. bool Invalid = false; + TemplateIdAnnotation *TemplateId = + D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId + ? D.getName().TemplateId + : nullptr; TemplateParameterList *TemplateParams = MatchTemplateParametersToScopeSpecifier( D.getDeclSpec().getBeginLoc(), D.getIdentifierLoc(), - D.getCXXScopeSpec(), - D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId - ? D.getName().TemplateId - : nullptr, - TemplateParamLists, isFriend, isMemberSpecialization, - Invalid); + D.getCXXScopeSpec(), TemplateId, TemplateParamLists, isFriend, + isMemberSpecialization, Invalid); if (TemplateParams) { // Check that we can declare a template here. if (CheckTemplateDeclScope(S, TemplateParams)) @@ -9921,6 +9921,11 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, if (Name.getNameKind() == DeclarationName::CXXDestructorName) { Diag(NewFD->getLocation(), diag::err_destructor_template); NewFD->setInvalidDecl(); + // Function template with explicit template arguments. + } else if (TemplateId) { + Diag(D.getIdentifierLoc(), diag::err_function_template_partial_spec) + << SourceRange(TemplateId->LAngleLoc, TemplateId->RAngleLoc); + NewFD->setInvalidDecl(); } // If we're adding a template to a dependent context, we may need to @@ -9973,6 +9978,11 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, << FixItHint::CreateRemoval(RemoveRange) << FixItHint::CreateInsertion(InsertLoc, "<>"); Invalid = true; + + // Recover by faking up an empty template argument list. + HasExplicitTemplateArgs = true; + TemplateArgs.setLAngleLoc(InsertLoc); + TemplateArgs.setRAngleLoc(InsertLoc); } } } else { @@ -9986,6 +9996,33 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, if (TemplateParamLists.size() > 0) // For source fidelity, store all the template param lists. NewFD->setTemplateParameterListsInfo(Context, TemplateParamLists); + + // "friend void foo<>(int);" is an implicit specialization decl. + if (isFriend && TemplateId) + isFunctionTemplateSpecialization = true; + } + + // If this is a function template specialization and the unqualified-id of + // the declarator-id is a template-id, convert the template argument list + // into our AST format and check for unexpanded packs. + if (isFunctionTemplateSpecialization && TemplateId) { + HasExplicitTemplateArgs = true; + + TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc); + TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc); + ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(), + TemplateId->NumArgs); + translateTemplateArguments(TemplateArgsPtr, TemplateArgs); + + // FIXME: Should we check for unexpanded packs if this was an (invalid) + // declaration of a function template partial specialization? Should we + // consider the unexpanded pack context to be a partial specialization? + for (const TemplateArgumentLoc &ArgLoc : TemplateArgs.arguments()) { + if (DiagnoseUnexpandedParameterPack( + ArgLoc, isFriend ? UPPC_FriendDeclaration + : UPPC_ExplicitSpecialization)) + NewFD->setInvalidDecl(); + } } if (Invalid) { @@ -10438,46 +10475,6 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, diag::ext_operator_new_delete_declared_inline) << NewFD->getDeclName(); - // If the declarator is a template-id, translate the parser's template - // argument list into our AST format. - if (D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId) { - TemplateIdAnnotation *TemplateId = D.getName().TemplateId; - TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc); - TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc); - ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(), - TemplateId->NumArgs); - translateTemplateArguments(TemplateArgsPtr, - TemplateArgs); - - HasExplicitTemplateArgs = true; - - if (NewFD->isInvalidDecl()) { - HasExplicitTemplateArgs = false; - } else if (FunctionTemplate) { - // Function template with explicit template arguments. - Diag(D.getIdentifierLoc(), diag::err_function_template_partial_spec) - << SourceRange(TemplateId->LAngleLoc, TemplateId->RAngleLoc); - - HasExplicitTemplateArgs = false; - } else if (isFriend) { - // "friend void foo<>(int);" is an implicit specialization decl. - isFunctionTemplateSpecialization = true; - } else { - assert(isFunctionTemplateSpecialization && - "should have a 'template<>' for this decl"); - } - } else if (isFriend && isFunctionTemplateSpecialization) { - // This combination is only possible in a recovery case; the user - // wrote something like: - // template <> friend void foo(int); - // which we're recovering from as if the user had written: - // friend void foo<>(int); - // Go ahead and fake up a template id. - HasExplicitTemplateArgs = true; - TemplateArgs.setLAngleLoc(D.getIdentifierLoc()); - TemplateArgs.setRAngleLoc(D.getIdentifierLoc()); - } - // We do not add HD attributes to specializations here because // they may have different constexpr-ness compared to their // templates and, after maybeAddCUDAHostDeviceAttrs() is applied, diff --git a/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp b/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp index 30ce6b40e1fb..3c500c2c4dc4 100644 --- a/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp +++ b/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp @@ -376,6 +376,11 @@ namespace Specializations { template struct PrimaryClass; // expected-error{{partial specialization contains unexpanded parameter pack 'Ts'}} + template + void PrimaryFunction(); + template + void PrimaryFunction(); // expected-error{{function template partial specialization is not allowed}} + #if __cplusplus >= 201402L template constexpr int PrimaryVar = 0; @@ -392,6 +397,13 @@ namespace Specializations { template struct InnerClass; // expected-error{{partial specialization contains unexpanded parameter pack 'Ts'}} + template + void InnerFunction(); + template<> + void InnerFunction(); // expected-error{{explicit specialization contains unexpanded parameter pack 'Ts'}} + + friend void PrimaryFunction(); // expected-error{{friend declaration contains unexpanded parameter pack 'Ts'}} + #if __cplusplus >= 201402L template constexpr static int InnerVar = 0; -- GitLab From 569ec185f5dc4a9e4a239948191977ecc2b2b475 Mon Sep 17 00:00:00 2001 From: Dmitry Vasilyev Date: Thu, 4 Jan 2024 20:42:51 +0400 Subject: [PATCH 005/728] [llvm-cxxfilt] Added the option --no-params (#75348) Added -p / --no-params flag to skip demangling function parameters similar to how it is supported by GNU c++filt tool. There are cases when users want to demangle a large number of symbols in bulk, for example, at startup, and do not care about function parameters and overloads at that time. Skipping the demangling of parameter types led to a measurable improvement in performance. Our users reported about 15% speed up with GNU c++filt and we expect similar results with llvm-cxxfilt with this patch. --- libcxxabi/src/demangle/ItaniumDemangle.h | 22 +++++++++---- llvm/docs/CommandGuide/llvm-cxxfilt.rst | 4 +++ llvm/include/llvm/Demangle/Demangle.h | 5 +-- llvm/include/llvm/Demangle/ItaniumDemangle.h | 22 +++++++++---- llvm/lib/Demangle/Demangle.cpp | 5 +-- llvm/lib/Demangle/ItaniumDemangle.cpp | 4 +-- llvm/test/tools/llvm-cxxfilt/no-params.test | 34 ++++++++++++++++++++ llvm/tools/llvm-cxxfilt/Opts.td | 2 ++ llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp | 10 ++++-- 9 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 llvm/test/tools/llvm-cxxfilt/no-params.test diff --git a/libcxxabi/src/demangle/ItaniumDemangle.h b/libcxxabi/src/demangle/ItaniumDemangle.h index 90f25519fad1..5a53a18bcc5f 100644 --- a/libcxxabi/src/demangle/ItaniumDemangle.h +++ b/libcxxabi/src/demangle/ItaniumDemangle.h @@ -2794,7 +2794,7 @@ template struct AbstractManglingParser { Node *parseClassEnumType(); Node *parseQualifiedType(); - Node *parseEncoding(); + Node *parseEncoding(bool ParseParams = true); bool parseCallOffset(); Node *parseSpecialName(); @@ -2911,7 +2911,7 @@ template struct AbstractManglingParser { Node *parseDestructorName(); /// Top-level entry point into the parser. - Node *parse(); + Node *parse(bool ParseParams = true); }; const char* parse_discriminator(const char* first, const char* last); @@ -5405,7 +5405,7 @@ Node *AbstractManglingParser::parseSpecialName() { // ::= // ::= template -Node *AbstractManglingParser::parseEncoding() { +Node *AbstractManglingParser::parseEncoding(bool ParseParams) { // The template parameters of an encoding are unrelated to those of the // enclosing context. SaveTemplateParams SaveTemplateParamsScope(this); @@ -5431,6 +5431,16 @@ Node *AbstractManglingParser::parseEncoding() { if (IsEndOfEncoding()) return Name; + // ParseParams may be false at the top level only, when called from parse(). + // For example in the mangled name _Z3fooILZ3BarEET_f, ParseParams may be + // false when demangling 3fooILZ3BarEET_f but is always true when demangling + // 3Bar. + if (!ParseParams) { + while (consume()) + ; + return Name; + } + Node *Attrs = nullptr; if (consumeIf("Ua9enable_ifI")) { size_t BeforeArgs = Names.size(); @@ -5895,9 +5905,9 @@ AbstractManglingParser::parseTemplateArgs(bool TagTemplates) { // extension ::= ___Z _block_invoke+ // extension ::= ___Z _block_invoke_+ template -Node *AbstractManglingParser::parse() { +Node *AbstractManglingParser::parse(bool ParseParams) { if (consumeIf("_Z") || consumeIf("__Z")) { - Node *Encoding = getDerived().parseEncoding(); + Node *Encoding = getDerived().parseEncoding(ParseParams); if (Encoding == nullptr) return nullptr; if (look() == '.') { @@ -5911,7 +5921,7 @@ Node *AbstractManglingParser::parse() { } if (consumeIf("___Z") || consumeIf("____Z")) { - Node *Encoding = getDerived().parseEncoding(); + Node *Encoding = getDerived().parseEncoding(ParseParams); if (Encoding == nullptr || !consumeIf("_block_invoke")) return nullptr; bool RequireNumber = consumeIf('_'); diff --git a/llvm/docs/CommandGuide/llvm-cxxfilt.rst b/llvm/docs/CommandGuide/llvm-cxxfilt.rst index 3f7deb171951..0933f0b5bed8 100644 --- a/llvm/docs/CommandGuide/llvm-cxxfilt.rst +++ b/llvm/docs/CommandGuide/llvm-cxxfilt.rst @@ -48,6 +48,10 @@ OPTIONS Print a summary of command line options. +.. option:: --no-params, -p + + Do not demangle function parameters or return types. + .. option:: --no-strip-underscore, -n Do not strip a leading underscore. This is the default for all platforms diff --git a/llvm/include/llvm/Demangle/Demangle.h b/llvm/include/llvm/Demangle/Demangle.h index 70cfc1418f0c..fe129603c078 100644 --- a/llvm/include/llvm/Demangle/Demangle.h +++ b/llvm/include/llvm/Demangle/Demangle.h @@ -32,7 +32,7 @@ enum : int { /// Returns a non-NULL pointer to a NUL-terminated C style string /// that should be explicitly freed, if successful. Otherwise, may return /// nullptr if mangled_name is not a valid mangling or is nullptr. -char *itaniumDemangle(std::string_view mangled_name); +char *itaniumDemangle(std::string_view mangled_name, bool ParseParams = true); enum MSDemangleFlags { MSDF_None = 0, @@ -68,7 +68,8 @@ char *dlangDemangle(std::string_view MangledName); std::string demangle(std::string_view MangledName); bool nonMicrosoftDemangle(std::string_view MangledName, std::string &Result, - bool CanHaveLeadingDot = true); + bool CanHaveLeadingDot = true, + bool ParseParams = true); /// "Partial" demangler. This supports demangling a string into an AST /// (typically an intermediate stage in itaniumDemangle) and querying certain diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h index e0ff035d47cf..06956f47c1f0 100644 --- a/llvm/include/llvm/Demangle/ItaniumDemangle.h +++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -2793,7 +2793,7 @@ template struct AbstractManglingParser { Node *parseClassEnumType(); Node *parseQualifiedType(); - Node *parseEncoding(); + Node *parseEncoding(bool ParseParams = true); bool parseCallOffset(); Node *parseSpecialName(); @@ -2910,7 +2910,7 @@ template struct AbstractManglingParser { Node *parseDestructorName(); /// Top-level entry point into the parser. - Node *parse(); + Node *parse(bool ParseParams = true); }; const char* parse_discriminator(const char* first, const char* last); @@ -5404,7 +5404,7 @@ Node *AbstractManglingParser::parseSpecialName() { // ::= // ::= template -Node *AbstractManglingParser::parseEncoding() { +Node *AbstractManglingParser::parseEncoding(bool ParseParams) { // The template parameters of an encoding are unrelated to those of the // enclosing context. SaveTemplateParams SaveTemplateParamsScope(this); @@ -5430,6 +5430,16 @@ Node *AbstractManglingParser::parseEncoding() { if (IsEndOfEncoding()) return Name; + // ParseParams may be false at the top level only, when called from parse(). + // For example in the mangled name _Z3fooILZ3BarEET_f, ParseParams may be + // false when demangling 3fooILZ3BarEET_f but is always true when demangling + // 3Bar. + if (!ParseParams) { + while (consume()) + ; + return Name; + } + Node *Attrs = nullptr; if (consumeIf("Ua9enable_ifI")) { size_t BeforeArgs = Names.size(); @@ -5894,9 +5904,9 @@ AbstractManglingParser::parseTemplateArgs(bool TagTemplates) { // extension ::= ___Z _block_invoke+ // extension ::= ___Z _block_invoke_+ template -Node *AbstractManglingParser::parse() { +Node *AbstractManglingParser::parse(bool ParseParams) { if (consumeIf("_Z") || consumeIf("__Z")) { - Node *Encoding = getDerived().parseEncoding(); + Node *Encoding = getDerived().parseEncoding(ParseParams); if (Encoding == nullptr) return nullptr; if (look() == '.') { @@ -5910,7 +5920,7 @@ Node *AbstractManglingParser::parse() { } if (consumeIf("___Z") || consumeIf("____Z")) { - Node *Encoding = getDerived().parseEncoding(); + Node *Encoding = getDerived().parseEncoding(ParseParams); if (Encoding == nullptr || !consumeIf("_block_invoke")) return nullptr; bool RequireNumber = consumeIf('_'); diff --git a/llvm/lib/Demangle/Demangle.cpp b/llvm/lib/Demangle/Demangle.cpp index 83f3cdc88c01..117b849d1c78 100644 --- a/llvm/lib/Demangle/Demangle.cpp +++ b/llvm/lib/Demangle/Demangle.cpp @@ -47,7 +47,8 @@ static bool isRustEncoding(std::string_view S) { return starts_with(S, "_R"); } static bool isDLangEncoding(std::string_view S) { return starts_with(S, "_D"); } bool llvm::nonMicrosoftDemangle(std::string_view MangledName, - std::string &Result, bool CanHaveLeadingDot) { + std::string &Result, bool CanHaveLeadingDot, + bool ParseParams) { char *Demangled = nullptr; // Do not consider the dot prefix as part of the demangled symbol name. @@ -57,7 +58,7 @@ bool llvm::nonMicrosoftDemangle(std::string_view MangledName, } if (isItaniumEncoding(MangledName)) - Demangled = itaniumDemangle(MangledName); + Demangled = itaniumDemangle(MangledName, ParseParams); else if (isRustEncoding(MangledName)) Demangled = rustDemangle(MangledName); else if (isDLangEncoding(MangledName)) diff --git a/llvm/lib/Demangle/ItaniumDemangle.cpp b/llvm/lib/Demangle/ItaniumDemangle.cpp index e3f208f0adf8..5c21b06a1d09 100644 --- a/llvm/lib/Demangle/ItaniumDemangle.cpp +++ b/llvm/lib/Demangle/ItaniumDemangle.cpp @@ -366,13 +366,13 @@ public: using Demangler = itanium_demangle::ManglingParser; -char *llvm::itaniumDemangle(std::string_view MangledName) { +char *llvm::itaniumDemangle(std::string_view MangledName, bool ParseParams) { if (MangledName.empty()) return nullptr; Demangler Parser(MangledName.data(), MangledName.data() + MangledName.length()); - Node *AST = Parser.parse(); + Node *AST = Parser.parse(ParseParams); if (!AST) return nullptr; diff --git a/llvm/test/tools/llvm-cxxfilt/no-params.test b/llvm/test/tools/llvm-cxxfilt/no-params.test new file mode 100644 index 000000000000..cf1eac27aa02 --- /dev/null +++ b/llvm/test/tools/llvm-cxxfilt/no-params.test @@ -0,0 +1,34 @@ +RUN: llvm-cxxfilt -n _Z3fooILZ3BarEET_f _Z3fooIPFcfEET_d _ZN1f2baC2ERKNS_2baIT_EE _Z3foov.123 | FileCheck %s --check-prefix=CHECK-PARAMS +RUN: llvm-cxxfilt -p -n _Z3fooILZ3BarEET_f _Z3fooIPFcfEET_d _ZN1f2baC2ERKNS_2baIT_EE _Z3foov.123 | FileCheck %s --check-prefix=CHECK-NO-PARAMS --match-full-lines +RUN: llvm-cxxfilt --no-params -n _Z3fooILZ3BarEET_f _Z3fooIPFcfEET_d _ZN1f2baC2ERKNS_2baIT_EE _Z3foov.123 | FileCheck %s --check-prefix=CHECK-NO-PARAMS --match-full-lines + +# Check that -p or --no-params flag omits function parameters and the return +# type. + +CHECK-PARAMS: Bar foo(float) +CHECK-NO-PARAMS: foo + +# Check that only the top-level function is impacted by the switch, and that +# nested function types in the encoding (e.g. where a function type is being +# used as a template parameter) still include their parameters. +# +# template T foo(double); +# typedef char (*F)(float); +# F foo(double) + +CHECK-PARAMS: char (*foo(double))(float) +CHECK-NO-PARAMS: foo + +# Use an invalid mangled name broken in the function parameters to check how -p +# or --no-params flag works. If the option is given we should be able to +# demangle the function name just fine. If it is not given, demangling will fail +# because of the invalid params. + +CHECK-PARAMS: _ZN1f2baC2ERKNS_2baIT_EE +CHECK-NO-PARAMS: f::ba::ba + +# Check that a vendor specific suffix is also omitted when --no-params is +# specified. This matches c++filt's behaviour. + +CHECK-PARAMS: foo() (.123) +CHECK-NO-PARAMS: foo diff --git a/llvm/tools/llvm-cxxfilt/Opts.td b/llvm/tools/llvm-cxxfilt/Opts.td index f652a1a7f88b..034cb267aab8 100644 --- a/llvm/tools/llvm-cxxfilt/Opts.td +++ b/llvm/tools/llvm-cxxfilt/Opts.td @@ -17,6 +17,7 @@ multiclass Eq { def help : FF<"help", "Display this help">; defm strip_underscore : BB<"strip-underscore", "Strip the leading underscore", "Don't strip the leading underscore">; def types : FF<"types", "Attempt to demangle types as well as function names">; +def no_params : FF<"no-params", "Skip function parameters and return types">; def version : FF<"version", "Display the version">; defm : Eq<"format", "Specify mangling format. Currently ignored because only 'gnu' is supported">; @@ -25,4 +26,5 @@ def : F<"s", "Alias for --format">; def : F<"_", "Alias for --strip-underscore">, Alias; def : F<"h", "Alias for --help">, Alias; def : F<"n", "Alias for --no-strip-underscore">, Alias; +def : F<"p", "Alias for --no-params">, Alias; def : F<"t", "Alias for --types">, Alias; diff --git a/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp b/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp index 4b9d88a65066..26a1f2f4afeb 100644 --- a/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp +++ b/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp @@ -54,6 +54,7 @@ public: }; } // namespace +static bool ParseParams; static bool StripUnderscore; static bool Types; @@ -74,18 +75,19 @@ static std::string demangle(const std::string &Mangled) { } std::string Result; - if (nonMicrosoftDemangle(DecoratedStr, Result, CanHaveLeadingDot)) + if (nonMicrosoftDemangle(DecoratedStr, Result, CanHaveLeadingDot, + ParseParams)) return Result; std::string Prefix; char *Undecorated = nullptr; if (Types) - Undecorated = itaniumDemangle(DecoratedStr); + Undecorated = itaniumDemangle(DecoratedStr, ParseParams); if (!Undecorated && starts_with(DecoratedStr, "__imp_")) { Prefix = "import thunk for "; - Undecorated = itaniumDemangle(DecoratedStr.substr(6)); + Undecorated = itaniumDemangle(DecoratedStr.substr(6), ParseParams); } Result = Undecorated ? Prefix + Undecorated : Mangled; @@ -173,6 +175,8 @@ int llvm_cxxfilt_main(int argc, char **argv, const llvm::ToolContext &) { else StripUnderscore = Triple(sys::getProcessTriple()).isOSBinFormatMachO(); + ParseParams = !Args.hasArg(OPT_no_params); + Types = Args.hasArg(OPT_types); std::vector Decorated = Args.getAllArgValues(OPT_INPUT); -- GitLab From 8f8152091cae186b35f73331df13f36b0f905eb4 Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Thu, 4 Jan 2024 16:50:31 +0000 Subject: [PATCH 006/728] [Clang][SME] Add IsStreamingOrSVE2p1 (#75958) This patch adds IsStreamingOrSVE2p1 to the applicable builtins and a warning for when those builtins are not used in a streaming or sve2p1 function. --- clang/include/clang/Basic/arm_sve.td | 123 ++++++++-------- clang/include/clang/Basic/arm_sve_sme_incl.td | 1 + clang/lib/Basic/Targets/AArch64.h | 1 + clang/lib/Sema/SemaChecking.cpp | 17 ++- .../aarch64-sve2-intrinsics/acle_sve2_revd.c | 4 +- .../acle_sve2p1_bfmlsl.c | 8 +- .../acle_sve2p1_cntp.c | 27 ++-- .../acle_sve2p1_fclamp.c | 14 +- .../acle_sve2p1_ld1.c | 2 + .../acle_sve2p1_pext.c | 5 +- .../acle_sve2p1_pfalse.c | 14 +- .../acle_sve2p1_psel.c | 26 +++- .../acle_sve2p1_sclamp.c | 20 ++- .../acle_sve2p1_stnt1.c | 2 - .../acle_sve2p1_uclamp.c | 20 ++- .../acle_sve2p1_while_pn.c | 137 +++++++++--------- .../aarch64-sme2-intrinsics/acle_sme2_imm.cpp | 2 +- .../Sema/aarch64-sme2-sve2p1-diagnostics.c | 37 +++++ clang/utils/TableGen/SveEmitter.cpp | 3 + 19 files changed, 289 insertions(+), 174 deletions(-) create mode 100644 clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 91f62c4c7633..17ee82d9304f 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1296,9 +1296,9 @@ def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsT def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate]>; } -let TargetGuard = "sve2p1" in { - def SVCREATE_2_B : SInst<"svcreate2[_{d}]", "2dd", "Pc", MergeNone, "", [IsTupleCreate]>; - def SVCREATE_4_B : SInst<"svcreate4[_{d}]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate]>; +let TargetGuard = "sve2p1|sme2" in { + def SVCREATE_2_B : SInst<"svcreate2[_{d}]", "2dd", "Pc", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>; + def SVCREATE_4_B : SInst<"svcreate4[_{d}]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1321,7 +1321,7 @@ def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>; } -let TargetGuard = "sve2p1" in { +let TargetGuard = "sve2p1|sme2" in { def SVGET_2_B : SInst<"svget2[_{d}]", "d2i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>; def SVGET_4_B : SInst<"svget4[_{d}]", "d4i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>; @@ -1976,39 +1976,37 @@ def SVFMINQV: SInst<"svminqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminq } let TargetGuard = "sve2p1|sme2" in { -//FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available -def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>; -def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>; +def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_3>]>; +def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_1>]>; -def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; } multiclass MultiVecLoad { - // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) - def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - - def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; } let TargetGuard = "sve2p1|sme2" in { @@ -2017,24 +2015,23 @@ let TargetGuard = "sve2p1|sme2" in { } multiclass MultiVecStore { - // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) - def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - - def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; } let TargetGuard = "sve2p1|sme2" in { @@ -2051,21 +2048,20 @@ def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "Ui", MergeNone def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "f", MergeNone, "aarch64_sve_fdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; } -let TargetGuard = "sve2p1|sme" in { -def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; -def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; +let TargetGuard = "sve2p1|sme2" in { +def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [IsStreamingOrSVE2p1], []>; +def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [IsStreamingOrSVE2p1], []>; defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">; } let TargetGuard = "sve2p1|sme2" in { - //FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available - def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingCompatible], []>; + def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], []>; - def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingCompatible]>; + def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingOrSVE2p1]>; - def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingCompatible], []>; - def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<1, ImmCheck2_4_Mul2>]>; + def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingOrSVE2p1], []>; + def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck2_4_Mul2>]>; } let TargetGuard = "(sve2|sme2),b16b16" in { @@ -2326,10 +2322,9 @@ let TargetGuard = "sme2" in { let TargetGuard = "sve2p1|sme2" in { // == BFloat16 multiply-subtract == -// FIXME: Make all of these IsStreamingOrSVE2p1 once that is added - def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingCompatible], []>; - def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingCompatible], []>; + def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingOrSVE2p1], []>; + def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingOrSVE2p1], []>; - def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; - def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>; } diff --git a/clang/include/clang/Basic/arm_sve_sme_incl.td b/clang/include/clang/Basic/arm_sve_sme_incl.td index 0dba8493bad2..ad29864440c9 100644 --- a/clang/include/clang/Basic/arm_sve_sme_incl.td +++ b/clang/include/clang/Basic/arm_sve_sme_incl.td @@ -227,6 +227,7 @@ def IsPreservesZA : FlagType<0x10000000000>; def IsReadZA : FlagType<0x20000000000>; def IsWriteZA : FlagType<0x40000000000>; def IsReductionQV : FlagType<0x80000000000>; +def IsStreamingOrSVE2p1 : FlagType<0x80000000000>; // Use for intrinsics that are common between sme/sme2 and sve2p1. // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h class ImmCheckType { diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h index f0e0782e7abe..e38fa5af5659 100644 --- a/clang/lib/Basic/Targets/AArch64.h +++ b/clang/lib/Basic/Targets/AArch64.h @@ -50,6 +50,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo { bool HasMatMul = false; bool HasBFloat16 = false; bool HasSVE2 = false; + bool HasSVE2p1 = false; bool HasSVE2AES = false; bool HasSVE2SHA3 = false; bool HasSVE2SM4 = false; diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 3168d38dd66c..f13164dc0638 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -2998,7 +2998,12 @@ static QualType getNeonEltType(NeonTypeFlags Flags, ASTContext &Context, llvm_unreachable("Invalid NeonTypeFlag!"); } -enum ArmStreamingType { ArmNonStreaming, ArmStreaming, ArmStreamingCompatible }; +enum ArmStreamingType { + ArmNonStreaming, + ArmStreaming, + ArmStreamingCompatible, + ArmStreamingOrSVE2p1 +}; bool Sema::ParseSVEImmChecks( CallExpr *TheCall, SmallVector, 3> &ImmChecks) { @@ -3156,6 +3161,16 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, const FunctionDecl *FD, ArmStreamingType BuiltinType) { ArmStreamingType FnType = getArmStreamingFnType(FD); + if (BuiltinType == ArmStreamingOrSVE2p1) { + // Check intrinsics that are available in [sve2p1 or sme/sme2]. + llvm::StringMap CallerFeatureMap; + S.Context.getFunctionFeatureMap(CallerFeatureMap, FD); + if (Builtin::evaluateRequiredTargetFeatures("sve2p1", CallerFeatureMap)) + BuiltinType = ArmStreamingCompatible; + else + BuiltinType = ArmStreaming; + } + if (FnType == ArmStreaming && BuiltinType == ArmNonStreaming) { S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin) << TheCall->getSourceRange() << "streaming"; diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c index 74a90583a173..d82d69442b8f 100644 --- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c +++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c @@ -1,7 +1,7 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ -// RUN: -target-feature +sme -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ @@ -9,7 +9,7 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s #include #ifdef SVE_OVERLOADED_FORMS diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c index c1d14e16ad17..22d951c069bc 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c @@ -2,20 +2,20 @@ // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -target-feature -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include #ifndef TEST_SME2 #define ATTR #else -#define ATTR __arm_streaming_compatible +#define ATTR __arm_streaming #endif #ifdef SVE_OVERLOADED_FORMS diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c index 56b1d9926221..9bf55eaa6a08 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c @@ -3,10 +3,19 @@ // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svcntp_c8_vlx2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2) @@ -17,7 +26,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c8_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c8_vlx2(svcount_t pnn) ATTR { return svcntp_c8(pnn, 2); } @@ -31,7 +40,7 @@ uint64_t test_svcntp_c8_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c8_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c8_vlx4(svcount_t pnn) ATTR { return svcntp_c8(pnn, 4); } @@ -45,7 +54,7 @@ uint64_t test_svcntp_c8_vlx4(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c16_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c16_vlx2(svcount_t pnn) ATTR { return svcntp_c16(pnn, 2); } @@ -59,7 +68,7 @@ uint64_t test_svcntp_c16_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c16_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c16_vlx4(svcount_t pnn) ATTR { return svcntp_c16(pnn, 4); } @@ -73,7 +82,7 @@ uint64_t test_svcntp_c16_vlx4(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c32_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c32_vlx2(svcount_t pnn) ATTR { return svcntp_c32(pnn, 2); } @@ -87,7 +96,7 @@ uint64_t test_svcntp_c32_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c32_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c32_vlx4(svcount_t pnn) ATTR { return svcntp_c32(pnn, 4); } @@ -101,7 +110,7 @@ uint64_t test_svcntp_c32_vlx4(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c64_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c64_vlx2(svcount_t pnn) ATTR { return svcntp_c64(pnn, 2); } @@ -115,6 +124,6 @@ uint64_t test_svcntp_c64_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c64_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c64_vlx4(svcount_t pnn) ATTR { return svcntp_c64(pnn, 4); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c index 5d8c5b7b8a18..7687257701a6 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c @@ -11,10 +11,16 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve \ -// RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -32,7 +38,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv8f16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) { +svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _f16, , )(op1, op2, op3); } @@ -46,7 +52,7 @@ svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv4f32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) { +svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _f32, , )(op1, op2, op3); } @@ -60,7 +66,7 @@ svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv2f64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat64_t test_svclamp_f64(svfloat64_t op1, svfloat64_t op2, svfloat64_t op3) { +svfloat64_t test_svclamp_f64(svfloat64_t op1, svfloat64_t op2, svfloat64_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _f64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c index 6f1231e776aa..7657165d8b3f 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c @@ -1,6 +1,8 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wno-unknown-attributes -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c index a3206029019c..8f08b32618b0 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c @@ -1,8 +1,11 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -DTEST_SME2 -target-feature +sve -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c index 19993e541812..afdb038fb931 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c @@ -1,14 +1,20 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svpfalse_c( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( zeroinitializer) @@ -19,7 +25,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( zeroinitializer) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svpfalse_c(void) __arm_streaming_compatible +svcount_t test_svpfalse_c(void) ATTR { return svpfalse_c(); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c index 73b7b0347dd9..de3f6a9a57bf 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c @@ -10,9 +10,19 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu \ +// RUN: -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming_compatible +#endif + // CHECK-LABEL: @test_svpsel_lane_b8( // CHECK-NEXT: entry: // CHECK-NEXT: [[ADD:%.*]] = add i32 [[IDX:%.*]], 15 @@ -25,7 +35,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.psel.nxv16i1( [[P1:%.*]], [[P2:%.*]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b8(p1, p2, idx + 15); } @@ -43,7 +53,7 @@ svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) __arm_strea // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv8i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b16(p1, p2, idx + 7); } @@ -61,7 +71,7 @@ svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) __arm_stre // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv4i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b32(p1, p2, idx + 3); } @@ -79,7 +89,7 @@ svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) __arm_stre // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv2i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b64(p1, p2, idx + 1); } @@ -99,7 +109,7 @@ svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) __arm_stre // CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP1]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP2]] // -svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c8(p1, p2, idx + 15); } @@ -121,7 +131,7 @@ svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) __arm_str // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c16(p1, p2, idx + 7); } @@ -143,7 +153,7 @@ svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) __arm_st // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c32(p1, p2, idx + 3); } @@ -165,6 +175,6 @@ svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) __arm_st // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c64(p1, p2, idx + 1); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c index 8c63a7455c79..04869fd550ec 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c @@ -10,9 +10,21 @@ // RUN: -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -30,7 +42,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv16i8( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) { +svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s8, , )(op1, op2, op3); } @@ -44,7 +56,7 @@ svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv8i16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) { +svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s16, , )(op1, op2, op3); } @@ -58,7 +70,7 @@ svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv4i32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) { +svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s32, , )(op1, op2, op3); } @@ -72,7 +84,7 @@ svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv2i64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint64_t test_svclamp_s64(svint64_t op1, svint64_t op2, svint64_t op3) { +svint64_t test_svclamp_s64(svint64_t op1, svint64_t op2, svint64_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c index 0d8696a7634a..b1ca27b7b68a 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c @@ -505,11 +505,9 @@ void test_svstnt1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) ATTR return SVE_ACLE_FUNC(svstnt1,_f64_x4,,)(pn, base, v); } - // == VNUM variants == - // CHECK-LABEL: @test_svstnt1_vnum_u8_x2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[V:%.*]], i64 0) diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c index b87898624887..37bfd4265a43 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c @@ -10,9 +10,21 @@ // RUN: -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -30,7 +42,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv16i8( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) { +svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u8, , )(op1, op2, op3); } @@ -44,7 +56,7 @@ svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv8i16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) { +svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u16, , )(op1, op2, op3); } @@ -58,7 +70,7 @@ svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv4i32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) { +svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u32, , )(op1, op2, op3); } @@ -72,7 +84,7 @@ svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv2i64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint64_t test_svclamp_u64(svuint64_t op1, svuint64_t op2, svuint64_t op3) { +svuint64_t test_svclamp_u64(svuint64_t op1, svuint64_t op2, svuint64_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c index 143a43b4a921..11ebec9e7cbf 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c @@ -1,10 +1,10 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s // REQUIRES: aarch64-registered-target @@ -16,6 +16,11 @@ #define SVE_ACLE_FUNC(A1, A2) A1##A2 #endif +#ifdef TEST_SME2 +#define ATTR __arm_streaming +#else +#define ATTR +#endif // WHILEGE @@ -29,7 +34,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 2); } @@ -44,7 +49,7 @@ svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 4); } @@ -59,7 +64,7 @@ svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 2); } @@ -74,7 +79,7 @@ svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 4); } @@ -89,7 +94,7 @@ svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 2); } @@ -104,7 +109,7 @@ svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 4); } @@ -119,7 +124,7 @@ svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 2); } @@ -134,7 +139,7 @@ svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 4); } @@ -152,7 +157,7 @@ svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 2); } @@ -167,7 +172,7 @@ svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 4); } @@ -182,7 +187,7 @@ svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 2); } @@ -197,7 +202,7 @@ svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 4); } @@ -212,7 +217,7 @@ svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 2); } @@ -227,7 +232,7 @@ svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 4); } @@ -242,7 +247,7 @@ svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 2); } @@ -257,7 +262,7 @@ svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 4); } @@ -275,7 +280,7 @@ svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 2); } @@ -290,7 +295,7 @@ svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 4); } @@ -305,7 +310,7 @@ svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 2); } @@ -320,7 +325,7 @@ svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 4); } @@ -335,7 +340,7 @@ svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 2); } @@ -350,7 +355,7 @@ svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 4); } @@ -365,7 +370,7 @@ svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 2); } @@ -380,7 +385,7 @@ svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 4); } @@ -398,7 +403,7 @@ svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 2); } @@ -413,7 +418,7 @@ svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 4); } @@ -428,7 +433,7 @@ svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 2); } @@ -443,7 +448,7 @@ svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 4); } @@ -458,7 +463,7 @@ svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 2); } @@ -473,7 +478,7 @@ svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 4); } @@ -488,7 +493,7 @@ svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 2); } @@ -503,7 +508,7 @@ svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 4); } @@ -521,7 +526,7 @@ svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 2); } @@ -536,7 +541,7 @@ svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 4); } @@ -551,7 +556,7 @@ svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 2); } @@ -566,7 +571,7 @@ svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 4); } @@ -581,7 +586,7 @@ svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 2); } @@ -596,7 +601,7 @@ svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 4); } @@ -611,7 +616,7 @@ svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 2); } @@ -626,7 +631,7 @@ svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 4); } @@ -644,7 +649,7 @@ svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 2); } @@ -659,7 +664,7 @@ svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 4); } @@ -674,7 +679,7 @@ svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 2); } @@ -689,7 +694,7 @@ svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 4); } @@ -704,7 +709,7 @@ svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 2); } @@ -719,7 +724,7 @@ svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 4); } @@ -734,7 +739,7 @@ svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 2); } @@ -749,7 +754,7 @@ svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 4); } @@ -767,7 +772,7 @@ svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 2); } @@ -782,7 +787,7 @@ svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 4); } @@ -797,7 +802,7 @@ svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 2); } @@ -812,7 +817,7 @@ svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 4); } @@ -827,7 +832,7 @@ svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 2); } @@ -842,7 +847,7 @@ svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 4); } @@ -857,7 +862,7 @@ svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 2); } @@ -872,7 +877,7 @@ svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 4); } @@ -890,7 +895,7 @@ svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 2); } @@ -905,7 +910,7 @@ svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 4); } @@ -920,7 +925,7 @@ svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 2); } @@ -935,7 +940,7 @@ svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 4); } @@ -950,7 +955,7 @@ svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 2); } @@ -965,7 +970,7 @@ svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 4); } @@ -980,7 +985,7 @@ svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 2); } @@ -995,7 +1000,7 @@ svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 4); } diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp index 6a6370bf99b1..5118f743174c 100644 --- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp +++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp @@ -237,7 +237,7 @@ void test_svluti4_lane_zt_x2(svuint8_t zn_u8) __arm_streaming __arm_shared_za __ svluti4_lane_zt_f32_x2(0, zn_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} } -void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming_compatible { +void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming { svbfmlslb_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} svbfmlslt_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} } diff --git a/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c b/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c new file mode 100644 index 000000000000..4debc14190aa --- /dev/null +++ b/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c @@ -0,0 +1,37 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -fsyntax-only -verify %s + +// REQUIRES: aarch64-registered-target +#include "arm_sve.h" + +//svldnt1: + +__attribute__((target("+sme2"))) +svuint8x2_t sme2_or_sve2p1_intrinsic_test_sme2_invalid(svcount_t png, const uint8_t *rn) { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a non-streaming function}} + return svldnt1_u8_x2(png, rn); +} + +__attribute__((target("+sme2"))) +svint16x4_t sme2_or_sve2p1_intrinsic_test_sme2(svcount_t png, const int16_t *rn) __arm_streaming { + // expected-no-warning + return svldnt1_s16_x4(png, rn); +} + +__attribute__((target("+sve2p1"))) +svuint32x2_t sme2_or_sve2p1_intrinsic_test_sve2p1(svcount_t png, const uint32_t *rn) { + // expected-no-warning + return svldnt1_u32_x2(png, rn); +} + +__attribute__((target("+sme2,+sve2p1"))) +svint64x4_t sme2_or_sve2p1_intrinsic_test_both_arm_streaming(svcount_t png, const int64_t *rn) __arm_streaming { + // expected-no-warning + return svldnt1_s64_x4(png, rn); +} + +__attribute__((target("+sme2,+sve2p1"))) +svint64x4_t sme2_or_sve2p1_intrinsic_test_both_no_arm_streaming(svcount_t png, const int64_t *rn) { + // expected-no-warning + return svldnt1_s64_x4(png, rn); +} diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp index 6c302da106a2..5de2223e71b0 100644 --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -1773,11 +1773,14 @@ void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) { llvm::StringMap> StreamingMap; uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming"); + uint64_t IsStreamingOrSVE2p1Flag = getEnumValueForFlag("IsStreamingOrSVE2p1"); uint64_t IsStreamingCompatibleFlag = getEnumValueForFlag("IsStreamingCompatible"); for (auto &Def : Defs) { if (Def->isFlagSet(IsStreamingFlag)) StreamingMap["ArmStreaming"].insert(Def->getMangledName()); + else if (Def->isFlagSet(IsStreamingOrSVE2p1Flag)) + StreamingMap["ArmStreamingOrSVE2p1"].insert(Def->getMangledName()); else if (Def->isFlagSet(IsStreamingCompatibleFlag)) StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName()); else -- GitLab From a7a78fd427569a7ad8a27e682a66fe414f004a35 Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Thu, 4 Jan 2024 16:53:14 +0000 Subject: [PATCH 007/728] Revert "[Clang][SME] Add IsStreamingOrSVE2p1" (#76973) Reverts llvm/llvm-project#75958 I mistakenly included a commit from my local main after rebasing. --- clang/include/clang/Basic/arm_sve.td | 123 ++++++++-------- clang/include/clang/Basic/arm_sve_sme_incl.td | 1 - clang/lib/Basic/Targets/AArch64.h | 1 - clang/lib/Sema/SemaChecking.cpp | 17 +-- .../aarch64-sve2-intrinsics/acle_sve2_revd.c | 4 +- .../acle_sve2p1_bfmlsl.c | 8 +- .../acle_sve2p1_cntp.c | 27 ++-- .../acle_sve2p1_fclamp.c | 14 +- .../acle_sve2p1_ld1.c | 2 - .../acle_sve2p1_pext.c | 5 +- .../acle_sve2p1_pfalse.c | 14 +- .../acle_sve2p1_psel.c | 26 +--- .../acle_sve2p1_sclamp.c | 20 +-- .../acle_sve2p1_stnt1.c | 2 + .../acle_sve2p1_uclamp.c | 20 +-- .../acle_sve2p1_while_pn.c | 137 +++++++++--------- .../aarch64-sme2-intrinsics/acle_sme2_imm.cpp | 2 +- .../Sema/aarch64-sme2-sve2p1-diagnostics.c | 37 ----- clang/utils/TableGen/SveEmitter.cpp | 3 - 19 files changed, 174 insertions(+), 289 deletions(-) delete mode 100644 clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 17ee82d9304f..91f62c4c7633 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1296,9 +1296,9 @@ def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsT def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate]>; } -let TargetGuard = "sve2p1|sme2" in { - def SVCREATE_2_B : SInst<"svcreate2[_{d}]", "2dd", "Pc", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>; - def SVCREATE_4_B : SInst<"svcreate4[_{d}]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>; +let TargetGuard = "sve2p1" in { + def SVCREATE_2_B : SInst<"svcreate2[_{d}]", "2dd", "Pc", MergeNone, "", [IsTupleCreate]>; + def SVCREATE_4_B : SInst<"svcreate4[_{d}]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate]>; } //////////////////////////////////////////////////////////////////////////////// @@ -1321,7 +1321,7 @@ def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>; } -let TargetGuard = "sve2p1|sme2" in { +let TargetGuard = "sve2p1" in { def SVGET_2_B : SInst<"svget2[_{d}]", "d2i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>; def SVGET_4_B : SInst<"svget4[_{d}]", "d4i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>; @@ -1976,37 +1976,39 @@ def SVFMINQV: SInst<"svminqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminq } let TargetGuard = "sve2p1|sme2" in { -def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_3>]>; -def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_1>]>; +//FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available +def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>; +def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>; -def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; } multiclass MultiVecLoad { - def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - - def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; } let TargetGuard = "sve2p1|sme2" in { @@ -2015,23 +2017,24 @@ let TargetGuard = "sve2p1|sme2" in { } multiclass MultiVecStore { - def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - - def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; } let TargetGuard = "sve2p1|sme2" in { @@ -2048,20 +2051,21 @@ def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "Ui", MergeNone def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "f", MergeNone, "aarch64_sve_fdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; } -let TargetGuard = "sve2p1|sme2" in { -def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [IsStreamingOrSVE2p1], []>; -def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [IsStreamingOrSVE2p1], []>; +let TargetGuard = "sve2p1|sme" in { +def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; +def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">; } let TargetGuard = "sve2p1|sme2" in { - def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], []>; + //FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available + def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingCompatible], []>; - def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingOrSVE2p1]>; + def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingCompatible]>; - def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingOrSVE2p1], []>; - def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck2_4_Mul2>]>; + def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingCompatible], []>; + def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<1, ImmCheck2_4_Mul2>]>; } let TargetGuard = "(sve2|sme2),b16b16" in { @@ -2322,9 +2326,10 @@ let TargetGuard = "sme2" in { let TargetGuard = "sve2p1|sme2" in { // == BFloat16 multiply-subtract == - def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingOrSVE2p1], []>; - def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingOrSVE2p1], []>; +// FIXME: Make all of these IsStreamingOrSVE2p1 once that is added + def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingCompatible], []>; + def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingCompatible], []>; - def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>; - def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; } diff --git a/clang/include/clang/Basic/arm_sve_sme_incl.td b/clang/include/clang/Basic/arm_sve_sme_incl.td index ad29864440c9..0dba8493bad2 100644 --- a/clang/include/clang/Basic/arm_sve_sme_incl.td +++ b/clang/include/clang/Basic/arm_sve_sme_incl.td @@ -227,7 +227,6 @@ def IsPreservesZA : FlagType<0x10000000000>; def IsReadZA : FlagType<0x20000000000>; def IsWriteZA : FlagType<0x40000000000>; def IsReductionQV : FlagType<0x80000000000>; -def IsStreamingOrSVE2p1 : FlagType<0x80000000000>; // Use for intrinsics that are common between sme/sme2 and sve2p1. // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h class ImmCheckType { diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h index e38fa5af5659..f0e0782e7abe 100644 --- a/clang/lib/Basic/Targets/AArch64.h +++ b/clang/lib/Basic/Targets/AArch64.h @@ -50,7 +50,6 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo { bool HasMatMul = false; bool HasBFloat16 = false; bool HasSVE2 = false; - bool HasSVE2p1 = false; bool HasSVE2AES = false; bool HasSVE2SHA3 = false; bool HasSVE2SM4 = false; diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index f13164dc0638..3168d38dd66c 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -2998,12 +2998,7 @@ static QualType getNeonEltType(NeonTypeFlags Flags, ASTContext &Context, llvm_unreachable("Invalid NeonTypeFlag!"); } -enum ArmStreamingType { - ArmNonStreaming, - ArmStreaming, - ArmStreamingCompatible, - ArmStreamingOrSVE2p1 -}; +enum ArmStreamingType { ArmNonStreaming, ArmStreaming, ArmStreamingCompatible }; bool Sema::ParseSVEImmChecks( CallExpr *TheCall, SmallVector, 3> &ImmChecks) { @@ -3161,16 +3156,6 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, const FunctionDecl *FD, ArmStreamingType BuiltinType) { ArmStreamingType FnType = getArmStreamingFnType(FD); - if (BuiltinType == ArmStreamingOrSVE2p1) { - // Check intrinsics that are available in [sve2p1 or sme/sme2]. - llvm::StringMap CallerFeatureMap; - S.Context.getFunctionFeatureMap(CallerFeatureMap, FD); - if (Builtin::evaluateRequiredTargetFeatures("sve2p1", CallerFeatureMap)) - BuiltinType = ArmStreamingCompatible; - else - BuiltinType = ArmStreaming; - } - if (FnType == ArmStreaming && BuiltinType == ArmNonStreaming) { S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin) << TheCall->getSourceRange() << "streaming"; diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c index d82d69442b8f..74a90583a173 100644 --- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c +++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c @@ -1,7 +1,7 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ -// RUN: -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: -target-feature +sme -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ @@ -9,7 +9,7 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s #include #ifdef SVE_OVERLOADED_FORMS diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c index 22d951c069bc..c1d14e16ad17 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c @@ -2,20 +2,20 @@ // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -target-feature -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include #ifndef TEST_SME2 #define ATTR #else -#define ATTR __arm_streaming +#define ATTR __arm_streaming_compatible #endif #ifdef SVE_OVERLOADED_FORMS diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c index 9bf55eaa6a08..56b1d9926221 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c @@ -3,19 +3,10 @@ // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include -#ifndef TEST_SME2 -#define ATTR -#else -#define ATTR __arm_streaming -#endif - // CHECK-LABEL: @test_svcntp_c8_vlx2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2) @@ -26,7 +17,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c8_vlx2(svcount_t pnn) ATTR { +uint64_t test_svcntp_c8_vlx2(svcount_t pnn) { return svcntp_c8(pnn, 2); } @@ -40,7 +31,7 @@ uint64_t test_svcntp_c8_vlx2(svcount_t pnn) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c8_vlx4(svcount_t pnn) ATTR { +uint64_t test_svcntp_c8_vlx4(svcount_t pnn) { return svcntp_c8(pnn, 4); } @@ -54,7 +45,7 @@ uint64_t test_svcntp_c8_vlx4(svcount_t pnn) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c16_vlx2(svcount_t pnn) ATTR { +uint64_t test_svcntp_c16_vlx2(svcount_t pnn) { return svcntp_c16(pnn, 2); } @@ -68,7 +59,7 @@ uint64_t test_svcntp_c16_vlx2(svcount_t pnn) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c16_vlx4(svcount_t pnn) ATTR { +uint64_t test_svcntp_c16_vlx4(svcount_t pnn) { return svcntp_c16(pnn, 4); } @@ -82,7 +73,7 @@ uint64_t test_svcntp_c16_vlx4(svcount_t pnn) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c32_vlx2(svcount_t pnn) ATTR { +uint64_t test_svcntp_c32_vlx2(svcount_t pnn) { return svcntp_c32(pnn, 2); } @@ -96,7 +87,7 @@ uint64_t test_svcntp_c32_vlx2(svcount_t pnn) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c32_vlx4(svcount_t pnn) ATTR { +uint64_t test_svcntp_c32_vlx4(svcount_t pnn) { return svcntp_c32(pnn, 4); } @@ -110,7 +101,7 @@ uint64_t test_svcntp_c32_vlx4(svcount_t pnn) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c64_vlx2(svcount_t pnn) ATTR { +uint64_t test_svcntp_c64_vlx2(svcount_t pnn) { return svcntp_c64(pnn, 2); } @@ -124,6 +115,6 @@ uint64_t test_svcntp_c64_vlx2(svcount_t pnn) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c64_vlx4(svcount_t pnn) ATTR { +uint64_t test_svcntp_c64_vlx4(svcount_t pnn) { return svcntp_c64(pnn, 4); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c index 7687257701a6..5d8c5b7b8a18 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c @@ -11,16 +11,10 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve \ -// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include -#ifndef TEST_SME2 -#define ATTR -#else -#define ATTR __arm_streaming -#endif - #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -38,7 +32,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv8f16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) ATTR { +svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) { return SVE_ACLE_FUNC(svclamp, _f16, , )(op1, op2, op3); } @@ -52,7 +46,7 @@ svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv4f32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) ATTR { +svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) { return SVE_ACLE_FUNC(svclamp, _f32, , )(op1, op2, op3); } @@ -66,7 +60,7 @@ svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv2f64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat64_t test_svclamp_f64(svfloat64_t op1, svfloat64_t op2, svfloat64_t op3) ATTR { +svfloat64_t test_svclamp_f64(svfloat64_t op1, svfloat64_t op2, svfloat64_t op3) { return SVE_ACLE_FUNC(svclamp, _f64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c index 7657165d8b3f..6f1231e776aa 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c @@ -1,8 +1,6 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wno-unknown-attributes -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c index 8f08b32618b0..a3206029019c 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c @@ -1,11 +1,8 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -DTEST_SME2 -target-feature +sve -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c index afdb038fb931..19993e541812 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c @@ -1,20 +1,14 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include -#ifndef TEST_SME2 -#define ATTR -#else -#define ATTR __arm_streaming -#endif - // CHECK-LABEL: @test_svpfalse_c( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( zeroinitializer) @@ -25,7 +19,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( zeroinitializer) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svpfalse_c(void) ATTR +svcount_t test_svpfalse_c(void) __arm_streaming_compatible { return svpfalse_c(); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c index de3f6a9a57bf..73b7b0347dd9 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c @@ -10,19 +10,9 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu \ -// RUN: -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include -#ifndef TEST_SME2 -#define ATTR -#else -#define ATTR __arm_streaming_compatible -#endif - // CHECK-LABEL: @test_svpsel_lane_b8( // CHECK-NEXT: entry: // CHECK-NEXT: [[ADD:%.*]] = add i32 [[IDX:%.*]], 15 @@ -35,7 +25,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.psel.nxv16i1( [[P1:%.*]], [[P2:%.*]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { +svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b8(p1, p2, idx + 15); } @@ -53,7 +43,7 @@ svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv8i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { +svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b16(p1, p2, idx + 7); } @@ -71,7 +61,7 @@ svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv4i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { +svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b32(p1, p2, idx + 3); } @@ -89,7 +79,7 @@ svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv2i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { +svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b64(p1, p2, idx + 1); } @@ -109,7 +99,7 @@ svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { // CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP1]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP2]] // -svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { +svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c8(p1, p2, idx + 15); } @@ -131,7 +121,7 @@ svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { +svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c16(p1, p2, idx + 7); } @@ -153,7 +143,7 @@ svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { +svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c32(p1, p2, idx + 3); } @@ -175,6 +165,6 @@ svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { +svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c64(p1, p2, idx + 1); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c index 04869fd550ec..8c63a7455c79 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c @@ -10,21 +10,9 @@ // RUN: -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ -// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ -// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ -// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include -#ifndef TEST_SME2 -#define ATTR -#else -#define ATTR __arm_streaming -#endif - #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -42,7 +30,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv16i8( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) ATTR { +svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) { return SVE_ACLE_FUNC(svclamp, _s8, , )(op1, op2, op3); } @@ -56,7 +44,7 @@ svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv8i16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) ATTR { +svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) { return SVE_ACLE_FUNC(svclamp, _s16, , )(op1, op2, op3); } @@ -70,7 +58,7 @@ svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv4i32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) ATTR { +svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) { return SVE_ACLE_FUNC(svclamp, _s32, , )(op1, op2, op3); } @@ -84,7 +72,7 @@ svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv2i64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint64_t test_svclamp_s64(svint64_t op1, svint64_t op2, svint64_t op3) ATTR { +svint64_t test_svclamp_s64(svint64_t op1, svint64_t op2, svint64_t op3) { return SVE_ACLE_FUNC(svclamp, _s64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c index b1ca27b7b68a..0d8696a7634a 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c @@ -505,9 +505,11 @@ void test_svstnt1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) ATTR return SVE_ACLE_FUNC(svstnt1,_f64_x4,,)(pn, base, v); } + // == VNUM variants == + // CHECK-LABEL: @test_svstnt1_vnum_u8_x2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[V:%.*]], i64 0) diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c index 37bfd4265a43..b87898624887 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c @@ -10,21 +10,9 @@ // RUN: -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ -// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ -// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ -// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include -#ifndef TEST_SME2 -#define ATTR -#else -#define ATTR __arm_streaming -#endif - #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -42,7 +30,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv16i8( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) ATTR { +svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) { return SVE_ACLE_FUNC(svclamp, _u8, , )(op1, op2, op3); } @@ -56,7 +44,7 @@ svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) ATTR { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv8i16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) ATTR { +svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) { return SVE_ACLE_FUNC(svclamp, _u16, , )(op1, op2, op3); } @@ -70,7 +58,7 @@ svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv4i32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) ATTR { +svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) { return SVE_ACLE_FUNC(svclamp, _u32, , )(op1, op2, op3); } @@ -84,7 +72,7 @@ svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv2i64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint64_t test_svclamp_u64(svuint64_t op1, svuint64_t op2, svuint64_t op3) ATTR { +svuint64_t test_svclamp_u64(svuint64_t op1, svuint64_t op2, svuint64_t op3) { return SVE_ACLE_FUNC(svclamp, _u64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c index 11ebec9e7cbf..143a43b4a921 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c @@ -1,10 +1,10 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s // REQUIRES: aarch64-registered-target @@ -16,11 +16,6 @@ #define SVE_ACLE_FUNC(A1, A2) A1##A2 #endif -#ifdef TEST_SME2 -#define ATTR __arm_streaming -#else -#define ATTR -#endif // WHILEGE @@ -34,7 +29,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 2); } @@ -49,7 +44,7 @@ svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 4); } @@ -64,7 +59,7 @@ svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 2); } @@ -79,7 +74,7 @@ svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 4); } @@ -94,7 +89,7 @@ svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 2); } @@ -109,7 +104,7 @@ svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 4); } @@ -124,7 +119,7 @@ svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 2); } @@ -139,7 +134,7 @@ svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 4); } @@ -157,7 +152,7 @@ svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 2); } @@ -172,7 +167,7 @@ svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 4); } @@ -187,7 +182,7 @@ svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 2); } @@ -202,7 +197,7 @@ svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 4); } @@ -217,7 +212,7 @@ svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 2); } @@ -232,7 +227,7 @@ svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 4); } @@ -247,7 +242,7 @@ svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 2); } @@ -262,7 +257,7 @@ svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 4); } @@ -280,7 +275,7 @@ svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 2); } @@ -295,7 +290,7 @@ svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 4); } @@ -310,7 +305,7 @@ svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 2); } @@ -325,7 +320,7 @@ svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 4); } @@ -340,7 +335,7 @@ svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 2); } @@ -355,7 +350,7 @@ svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 4); } @@ -370,7 +365,7 @@ svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 2); } @@ -385,7 +380,7 @@ svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 4); } @@ -403,7 +398,7 @@ svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 2); } @@ -418,7 +413,7 @@ svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 4); } @@ -433,7 +428,7 @@ svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 2); } @@ -448,7 +443,7 @@ svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 4); } @@ -463,7 +458,7 @@ svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 2); } @@ -478,7 +473,7 @@ svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 4); } @@ -493,7 +488,7 @@ svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 2); } @@ -508,7 +503,7 @@ svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 4); } @@ -526,7 +521,7 @@ svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 2); } @@ -541,7 +536,7 @@ svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 4); } @@ -556,7 +551,7 @@ svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 2); } @@ -571,7 +566,7 @@ svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 4); } @@ -586,7 +581,7 @@ svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 2); } @@ -601,7 +596,7 @@ svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 4); } @@ -616,7 +611,7 @@ svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 2); } @@ -631,7 +626,7 @@ svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 4); } @@ -649,7 +644,7 @@ svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 2); } @@ -664,7 +659,7 @@ svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 4); } @@ -679,7 +674,7 @@ svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 2); } @@ -694,7 +689,7 @@ svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 4); } @@ -709,7 +704,7 @@ svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 2); } @@ -724,7 +719,7 @@ svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 4); } @@ -739,7 +734,7 @@ svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 2); } @@ -754,7 +749,7 @@ svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 4); } @@ -772,7 +767,7 @@ svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 2); } @@ -787,7 +782,7 @@ svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 4); } @@ -802,7 +797,7 @@ svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 2); } @@ -817,7 +812,7 @@ svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 4); } @@ -832,7 +827,7 @@ svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 2); } @@ -847,7 +842,7 @@ svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 4); } @@ -862,7 +857,7 @@ svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 2); } @@ -877,7 +872,7 @@ svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) ATTR +svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) { return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 4); } @@ -895,7 +890,7 @@ svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 2); } @@ -910,7 +905,7 @@ svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 4); } @@ -925,7 +920,7 @@ svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 2); } @@ -940,7 +935,7 @@ svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 4); } @@ -955,7 +950,7 @@ svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 2); } @@ -970,7 +965,7 @@ svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 4); } @@ -985,7 +980,7 @@ svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 2); } @@ -1000,7 +995,7 @@ svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) ATTR // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2) ATTR +svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2) { return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 4); } diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp index 5118f743174c..6a6370bf99b1 100644 --- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp +++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp @@ -237,7 +237,7 @@ void test_svluti4_lane_zt_x2(svuint8_t zn_u8) __arm_streaming __arm_shared_za __ svluti4_lane_zt_f32_x2(0, zn_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} } -void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming { +void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming_compatible { svbfmlslb_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} svbfmlslt_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} } diff --git a/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c b/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c deleted file mode 100644 index 4debc14190aa..000000000000 --- a/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c +++ /dev/null @@ -1,37 +0,0 @@ -// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -fsyntax-only -verify %s - -// REQUIRES: aarch64-registered-target -#include "arm_sve.h" - -//svldnt1: - -__attribute__((target("+sme2"))) -svuint8x2_t sme2_or_sve2p1_intrinsic_test_sme2_invalid(svcount_t png, const uint8_t *rn) { - // expected-warning@+1 {{builtin call has undefined behaviour when called from a non-streaming function}} - return svldnt1_u8_x2(png, rn); -} - -__attribute__((target("+sme2"))) -svint16x4_t sme2_or_sve2p1_intrinsic_test_sme2(svcount_t png, const int16_t *rn) __arm_streaming { - // expected-no-warning - return svldnt1_s16_x4(png, rn); -} - -__attribute__((target("+sve2p1"))) -svuint32x2_t sme2_or_sve2p1_intrinsic_test_sve2p1(svcount_t png, const uint32_t *rn) { - // expected-no-warning - return svldnt1_u32_x2(png, rn); -} - -__attribute__((target("+sme2,+sve2p1"))) -svint64x4_t sme2_or_sve2p1_intrinsic_test_both_arm_streaming(svcount_t png, const int64_t *rn) __arm_streaming { - // expected-no-warning - return svldnt1_s64_x4(png, rn); -} - -__attribute__((target("+sme2,+sve2p1"))) -svint64x4_t sme2_or_sve2p1_intrinsic_test_both_no_arm_streaming(svcount_t png, const int64_t *rn) { - // expected-no-warning - return svldnt1_s64_x4(png, rn); -} diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp index 5de2223e71b0..6c302da106a2 100644 --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -1773,14 +1773,11 @@ void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) { llvm::StringMap> StreamingMap; uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming"); - uint64_t IsStreamingOrSVE2p1Flag = getEnumValueForFlag("IsStreamingOrSVE2p1"); uint64_t IsStreamingCompatibleFlag = getEnumValueForFlag("IsStreamingCompatible"); for (auto &Def : Defs) { if (Def->isFlagSet(IsStreamingFlag)) StreamingMap["ArmStreaming"].insert(Def->getMangledName()); - else if (Def->isFlagSet(IsStreamingOrSVE2p1Flag)) - StreamingMap["ArmStreamingOrSVE2p1"].insert(Def->getMangledName()); else if (Def->isFlagSet(IsStreamingCompatibleFlag)) StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName()); else -- GitLab From 917b404e2ccdcc31d2d64971ad094b80967a240b Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Thu, 4 Jan 2024 09:04:05 -0800 Subject: [PATCH 008/728] Add support for inline DWARF source files. (#75880) LLVM supports DWARF 5 linetable extension to store source files inline in DWARF. This is particularly useful for compiler-generated source code. This implementation tries to materialize them as temporary files lazily, so SBAPI clients don't need to be aware of them. rdar://110926168 --- lldb/include/lldb/Symbol/CompileUnit.h | 23 ++-- lldb/include/lldb/Symbol/SymbolFile.h | 2 +- lldb/include/lldb/Symbol/SymbolFileOnDemand.h | 2 +- lldb/include/lldb/Utility/FileSpecList.h | 106 ++++++++++++----- lldb/source/API/SBCompileUnit.cpp | 2 +- lldb/source/Commands/CommandObjectSource.cpp | 2 +- lldb/source/Core/ModuleList.cpp | 2 +- .../Clang/ClangUserExpression.cpp | 12 +- .../Clang/CppModuleConfiguration.cpp | 6 +- .../Breakpad/SymbolFileBreakpad.cpp | 5 +- .../SymbolFile/Breakpad/SymbolFileBreakpad.h | 3 +- .../Plugins/SymbolFile/CTF/SymbolFileCTF.h | 2 +- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 107 ++++++++++++------ .../SymbolFile/DWARF/SymbolFileDWARF.h | 9 +- .../DWARF/SymbolFileDWARFDebugMap.cpp | 4 +- .../DWARF/SymbolFileDWARFDebugMap.h | 2 +- .../Plugins/SymbolFile/JSON/SymbolFileJSON.h | 2 +- .../NativePDB/SymbolFileNativePDB.cpp | 4 +- .../NativePDB/SymbolFileNativePDB.h | 2 +- .../Plugins/SymbolFile/PDB/SymbolFilePDB.cpp | 2 +- .../Plugins/SymbolFile/PDB/SymbolFilePDB.h | 2 +- .../SymbolFile/Symtab/SymbolFileSymtab.cpp | 2 +- .../SymbolFile/Symtab/SymbolFileSymtab.h | 2 +- lldb/source/Symbol/CompileUnit.cpp | 13 +-- lldb/source/Symbol/SymbolFileOnDemand.cpp | 2 +- lldb/source/Utility/FileSpecList.cpp | 58 ++++++++-- .../inline-sourcefile/Makefile | 11 ++ .../TestInlineSourceFiles.py | 15 +++ .../inline-sourcefile/inline.ll | 39 +++++++ .../functionalities/inline-sourcefile/main.c | 7 ++ lldb/unittests/Core/FileSpecListTest.cpp | 8 +- 31 files changed, 335 insertions(+), 123 deletions(-) create mode 100644 lldb/test/API/functionalities/inline-sourcefile/Makefile create mode 100644 lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py create mode 100644 lldb/test/API/functionalities/inline-sourcefile/inline.ll create mode 100644 lldb/test/API/functionalities/inline-sourcefile/main.c diff --git a/lldb/include/lldb/Symbol/CompileUnit.h b/lldb/include/lldb/Symbol/CompileUnit.h index 93f191b49985..89e853ab599d 100644 --- a/lldb/include/lldb/Symbol/CompileUnit.h +++ b/lldb/include/lldb/Symbol/CompileUnit.h @@ -112,10 +112,13 @@ public: /// the compile unit is optimized will be made when /// CompileUnit::GetIsOptimized() is called. /// + /// \param[in] support_files + /// An rvalue list of already parsed support files. /// \see lldb::LanguageType CompileUnit(const lldb::ModuleSP &module_sp, void *user_data, const FileSpec &file_spec, lldb::user_id_t uid, - lldb::LanguageType language, lldb_private::LazyBool is_optimized); + lldb::LanguageType language, lldb_private::LazyBool is_optimized, + SupportFileList &&support_files = {}); /// Add a function to this compile unit. /// @@ -226,6 +229,9 @@ public: /// Return the primary source file associated with this compile unit. const FileSpec &GetPrimaryFile() const { return m_file_spec; } + /// Return the primary source file associated with this compile unit. + void SetPrimaryFile(const FileSpec &fs) { m_file_spec = fs; } + /// Get the line table for the compile unit. /// /// Called by clients and the SymbolFile plug-in. The SymbolFile plug-ins @@ -265,7 +271,13 @@ public: /// /// \return /// A support file list object. - const FileSpecList &GetSupportFiles(); + const SupportFileList &GetSupportFiles(); + + /// Used by plugins that parse the support file list. + SupportFileList &GetSupportFileList() { + m_flags.Set(flagsParsedSupportFiles); + return m_support_files; + } /// Get the compile unit's imported module list. /// @@ -331,8 +343,6 @@ public: /// A line table object pointer that this object now owns. void SetLineTable(LineTable *line_table); - void SetSupportFiles(FileSpecList support_files); - void SetDebugMacros(const DebugMacrosSP &debug_macros); /// Set accessor for the variable list. @@ -410,9 +420,8 @@ protected: std::vector m_imported_modules; /// The primary file associated with this compile unit. FileSpec m_file_spec; - /// Files associated with this compile unit's line table and - /// declarations. - FileSpecList m_support_files; + /// Files associated with this compile unit's line table and declarations. + SupportFileList m_support_files; /// Line table that will get parsed on demand. std::unique_ptr m_line_table_up; /// Debug macros that will get parsed on demand. diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h index c9a2a647a039..f356f7b789fa 100644 --- a/lldb/include/lldb/Symbol/SymbolFile.h +++ b/lldb/include/lldb/Symbol/SymbolFile.h @@ -197,7 +197,7 @@ public: return false; } virtual bool ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) = 0; + SupportFileList &support_files) = 0; virtual size_t ParseTypes(CompileUnit &comp_unit) = 0; virtual bool ParseIsOptimized(CompileUnit &comp_unit) { return false; } diff --git a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h index cde9f3c3b8ce..4e3009941aa7 100644 --- a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h +++ b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h @@ -81,7 +81,7 @@ public: llvm::function_ref) override; bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit, - lldb_private::FileSpecList &support_files) override; + lldb_private::SupportFileList &support_files) override; bool ParseIsOptimized(lldb_private::CompileUnit &comp_unit) override; diff --git a/lldb/include/lldb/Utility/FileSpecList.h b/lldb/include/lldb/Utility/FileSpecList.h index 77587aa91791..8cccb1949999 100644 --- a/lldb/include/lldb/Utility/FileSpecList.h +++ b/lldb/include/lldb/Utility/FileSpecList.h @@ -17,6 +17,86 @@ namespace lldb_private { class Stream; +/// Wraps either a FileSpec that represents a local file or a source +/// file whose contents is known (for example because it can be +/// reconstructed from debug info), but that hasn't been written to a +/// file yet. +class SupportFile { +protected: + FileSpec m_file_spec; + +public: + SupportFile(const FileSpec &spec) : m_file_spec(spec) {} + SupportFile(const SupportFile &other) = delete; + SupportFile(SupportFile &&other) = default; + virtual ~SupportFile() = default; + bool operator==(const SupportFile &other) { + return m_file_spec == other.m_file_spec; + } + /// Return the file name only. Useful for resolving breakpoints by file name. + const FileSpec &GetSpecOnly() const { return m_file_spec; }; + /// Materialize the file to disk and return the path to that temporary file. + virtual const FileSpec &Materialize() { return m_file_spec; } +}; + +/// A list of support files for a CompileUnit. +class SupportFileList { +public: + SupportFileList(){}; + SupportFileList(const SupportFileList &) = delete; + SupportFileList(SupportFileList &&other) = default; + + typedef std::vector> collection; + typedef collection::const_iterator const_iterator; + const_iterator begin() const { return m_files.begin(); } + const_iterator end() const { return m_files.end(); } + + void Append(const FileSpec &file) { + return Append(std::make_unique(file)); + } + void Append(std::unique_ptr &&file) { + m_files.push_back(std::move(file)); + } + // FIXME: Only used by SymbolFilePDB. Replace with a DenseSet at call site. + bool AppendIfUnique(const FileSpec &file); + size_t GetSize() const { return m_files.size(); } + const FileSpec &GetFileSpecAtIndex(size_t idx) const; + size_t FindFileIndex(size_t idx, const FileSpec &file, bool full) const; + /// Find a compatible file index. + /// + /// Find the index of a compatible file in the file spec list that matches \a + /// file starting \a idx entries into the file spec list. A file is considered + /// compatible if: + /// - The file matches exactly (only filename if \a file has no directory) + /// - If \a file is relative and any file in the list has this same suffix + /// - If any file in the list is relative and the relative path is a suffix + /// of \a file + /// + /// This is used to implement better matching for setting breakpoints in + /// source files where an IDE might specify a full path when setting the + /// breakpoint and debug info contains relative paths, if a user specifies + /// a relative path when setting a breakpoint. + /// + /// \param[in] idx + /// An index into the file list. + /// + /// \param[in] file + /// The file specification to search for. + /// + /// \return + /// The index of the file that matches \a file if it is found, + /// else UINT32_MAX is returned. + size_t FindCompatibleIndex(size_t idx, const FileSpec &file) const; + + template void EmplaceBack(Args &&...args) { + m_files.push_back( + std::make_unique(FileSpec(std::forward(args)...))); + } + +protected: + collection m_files; ///< A collection of FileSpec objects. +}; + /// \class FileSpecList FileSpecList.h "lldb/Utility/FileSpecList.h" /// A file collection class. /// @@ -114,32 +194,6 @@ public: /// else UINT32_MAX is returned. size_t FindFileIndex(size_t idx, const FileSpec &file, bool full) const; - /// Find a compatible file index. - /// - /// Find the index of a compatible file in the file spec list that matches \a - /// file starting \a idx entries into the file spec list. A file is considered - /// compatible if: - /// - The file matches exactly (only filename if \a file has no directory) - /// - If \a file is relative and any file in the list has this same suffix - /// - If any file in the list is relative and the relative path is a suffix - /// of \a file - /// - /// This is used to implement better matching for setting breakpoints in - /// source files where an IDE might specify a full path when setting the - /// breakpoint and debug info contains relative paths, if a user specifies - /// a relative path when setting a breakpoint. - /// - /// \param[in] idx - /// An index into the file list. - /// - /// \param[in] file - /// The file specification to search for. - /// - /// \return - /// The index of the file that matches \a file if it is found, - /// else UINT32_MAX is returned. - size_t FindCompatibleIndex(size_t idx, const FileSpec &file) const; - /// Get file at index. /// /// Gets a file from the file list. If \a idx is not a valid index, an empty diff --git a/lldb/source/API/SBCompileUnit.cpp b/lldb/source/API/SBCompileUnit.cpp index 3aa65e225d7a..65fdb11032b9 100644 --- a/lldb/source/API/SBCompileUnit.cpp +++ b/lldb/source/API/SBCompileUnit.cpp @@ -171,7 +171,7 @@ uint32_t SBCompileUnit::FindSupportFileIndex(uint32_t start_idx, LLDB_INSTRUMENT_VA(this, start_idx, sb_file, full); if (m_opaque_ptr) { - const FileSpecList &support_files = m_opaque_ptr->GetSupportFiles(); + const SupportFileList &support_files = m_opaque_ptr->GetSupportFiles(); return support_files.FindFileIndex(start_idx, sb_file.ref(), full); } return 0; diff --git a/lldb/source/Commands/CommandObjectSource.cpp b/lldb/source/Commands/CommandObjectSource.cpp index db158a7f5263..cabf6f0436f1 100644 --- a/lldb/source/Commands/CommandObjectSource.cpp +++ b/lldb/source/Commands/CommandObjectSource.cpp @@ -204,7 +204,7 @@ protected: if (cu) { assert(file_spec.GetFilename().AsCString()); bool has_path = (file_spec.GetDirectory().AsCString() != nullptr); - const FileSpecList &cu_file_list = cu->GetSupportFiles(); + const SupportFileList &cu_file_list = cu->GetSupportFiles(); size_t file_idx = cu_file_list.FindFileIndex(0, file_spec, has_path); if (file_idx != UINT32_MAX) { // Update the file to how it appears in the CU. diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp index aa89c93c8d05..2180f29f3694 100644 --- a/lldb/source/Core/ModuleList.cpp +++ b/lldb/source/Core/ModuleList.cpp @@ -164,7 +164,7 @@ void ModuleListProperties::UpdateSymlinkMappings() { llvm::sys::ScopedWriter lock(m_symlink_paths_mutex); const bool notify = false; m_symlink_paths.Clear(notify); - for (FileSpec symlink : list) { + for (auto symlink : list) { FileSpec resolved; Status status = FileSystem::Instance().Readlink(symlink, resolved); if (status.Success()) diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp index 68bdd96e8adb..30bc81c9ed8c 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp @@ -488,18 +488,18 @@ CppModuleConfiguration GetModuleConfig(lldb::LanguageType language, // Build a list of files we need to analyze to build the configuration. FileSpecList files; - for (const FileSpec &f : sc.comp_unit->GetSupportFiles()) - files.AppendIfUnique(f); + for (auto &f : sc.comp_unit->GetSupportFiles()) + files.AppendIfUnique(f->Materialize()); // We also need to look at external modules in the case of -gmodules as they // contain the support files for libc++ and the C library. llvm::DenseSet visited_symbol_files; sc.comp_unit->ForEachExternalModule( visited_symbol_files, [&files](Module &module) { for (std::size_t i = 0; i < module.GetNumCompileUnits(); ++i) { - const FileSpecList &support_files = + const SupportFileList &support_files = module.GetCompileUnitAtIndex(i)->GetSupportFiles(); - for (const FileSpec &f : support_files) { - files.AppendIfUnique(f); + for (auto &f : support_files) { + files.AppendIfUnique(f->Materialize()); } } return false; @@ -508,7 +508,7 @@ CppModuleConfiguration GetModuleConfig(lldb::LanguageType language, LLDB_LOG(log, "[C++ module config] Found {0} support files to analyze", files.GetSize()); if (log && log->GetVerbose()) { - for (const FileSpec &f : files) + for (auto &f : files) LLDB_LOGV(log, "[C++ module config] Analyzing support file: {0}", f.GetPath()); } diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp index 62443d1290dc..f43a04488230 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp @@ -134,9 +134,9 @@ bool CppModuleConfiguration::hasValidConfig() { CppModuleConfiguration::CppModuleConfiguration( const FileSpecList &support_files, const llvm::Triple &triple) { // Analyze all files we were given to build the configuration. - bool error = !llvm::all_of(support_files, - std::bind(&CppModuleConfiguration::analyzeFile, - this, std::placeholders::_1, triple)); + bool error = !llvm::all_of(support_files, [&](auto &file) { + return CppModuleConfiguration::analyzeFile(file, triple); + }); // If we have a valid configuration at this point, set the // include directories and module list that should be used. if (!error && hasValidConfig()) { diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp index 729d6af02402..47c8074adc5b 100644 --- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp +++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp @@ -278,13 +278,14 @@ bool SymbolFileBreakpad::ParseLineTable(CompileUnit &comp_unit) { } bool SymbolFileBreakpad::ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) { + SupportFileList &support_files) { std::lock_guard guard(GetModuleMutex()); CompUnitData &data = m_cu_data->GetEntryRef(comp_unit.GetID()).data; if (!data.support_files) ParseLineTableAndSupportFiles(comp_unit, data); - support_files = std::move(*data.support_files); + for (auto &fs : *data.support_files) + support_files.Append(fs); return true; } diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h index 214fbdd3ff3a..41e4e3b25801 100644 --- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h +++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h @@ -73,7 +73,7 @@ public: bool ParseDebugMacros(CompileUnit &comp_unit) override { return false; } bool ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) override; + SupportFileList &support_files) override; size_t ParseTypes(CompileUnit &cu) override { return 0; } bool ParseImportedModules( @@ -195,7 +195,6 @@ private: Bookmark bookmark; std::optional support_files; std::unique_ptr line_table_up; - }; uint32_t CalculateNumCompileUnits() override; diff --git a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h index f111937dbd6e..3a80138fffbc 100644 --- a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h +++ b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h @@ -66,7 +66,7 @@ public: bool ParseDebugMacros(CompileUnit &comp_unit) override { return false; } bool ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) override { + SupportFileList &support_files) override { return false; } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 505ea29ca4d4..447930ffe07b 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -10,6 +10,7 @@ #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/FileUtilities.h" #include "llvm/Support/Format.h" #include "llvm/Support/Threading.h" @@ -209,17 +210,14 @@ GetFileByIndex(const llvm::DWARFDebugLine::Prologue &prologue, size_t idx, return std::move(rel_path); } -static FileSpecList -ParseSupportFilesFromPrologue(const lldb::ModuleSP &module, - const llvm::DWARFDebugLine::Prologue &prologue, - FileSpec::Style style, - llvm::StringRef compile_dir = {}) { - FileSpecList support_files; - +static void ParseSupportFilesFromPrologue( + SupportFileList &support_files, const lldb::ModuleSP &module, + const llvm::DWARFDebugLine::Prologue &prologue, FileSpec::Style style, + llvm::StringRef compile_dir = {}) { // Handle the case where there are no files first to avoid having to special // case this later. if (prologue.FileNames.empty()) - return support_files; + return; // Before DWARF v5, the line table indexes were one based. const bool is_one_based = prologue.getVersion() < 5; @@ -235,6 +233,53 @@ ParseSupportFilesFromPrologue(const lldb::ModuleSP &module, for (size_t idx = first_file_idx; idx <= last_file_idx; ++idx) { std::string remapped_file; if (auto file_path = GetFileByIndex(prologue, idx, compile_dir, style)) { + auto entry = prologue.getFileNameEntry(idx); + auto source = entry.Source.getAsCString(); + if (!source) + consumeError(source.takeError()); + else { + llvm::StringRef source_ref(*source); + if (!source_ref.empty()) { + /// Wrap a path for an in-DWARF source file. Lazily write it + /// to disk when Materialize() is called. + struct LazyDWARFSourceFile : public SupportFile { + LazyDWARFSourceFile(const FileSpec &fs, llvm::StringRef source, + FileSpec::Style style) + : SupportFile(fs), source(source), style(style) {} + FileSpec tmp_file; + /// The file contents buffer. + llvm::StringRef source; + /// Deletes the temporary file at the end. + std::unique_ptr remover; + FileSpec::Style style; + + /// Write the file contents to a temporary file. + const FileSpec &Materialize() override { + if (tmp_file) + return tmp_file; + llvm::SmallString<0> name; + int fd; + auto orig_name = m_file_spec.GetFilename().GetStringRef(); + auto ec = llvm::sys::fs::createTemporaryFile( + "", llvm::sys::path::filename(orig_name, style), fd, name); + if (ec || fd <= 0) { + LLDB_LOG(GetLog(DWARFLog::DebugInfo), + "Could not create temporary file"); + return tmp_file; + } + remover = std::make_unique(name); + NativeFile file(fd, File::eOpenOptionWriteOnly, true); + size_t num_bytes = source.size(); + file.Write(source.data(), num_bytes); + tmp_file.SetPath(name); + return tmp_file; + } + }; + support_files.Append(std::make_unique( + FileSpec(*file_path), *source, style)); + continue; + } + } if (auto remapped = module->RemapSourceFile(llvm::StringRef(*file_path))) remapped_file = *remapped; else @@ -251,8 +296,6 @@ ParseSupportFilesFromPrologue(const lldb::ModuleSP &module, // Unconditionally add an entry, so the indices match up. support_files.EmplaceBack(remapped_file, style, checksum); } - - return support_files; } void SymbolFileDWARF::Initialize() { @@ -744,12 +787,13 @@ lldb::CompUnitSP SymbolFileDWARF::ParseCompileUnit(DWARFCompileUnit &dwarf_cu) { ModuleSP module_sp(m_objfile_sp->GetModule()); if (module_sp) { auto initialize_cu = [&](const FileSpec &file_spec, - LanguageType cu_language) { + LanguageType cu_language, + SupportFileList &&support_files = {}) { BuildCuTranslationTable(); cu_sp = std::make_shared( module_sp, &dwarf_cu, file_spec, *GetDWARFUnitIndex(dwarf_cu.GetID()), cu_language, - eLazyBoolCalculate); + eLazyBoolCalculate, std::move(support_files)); dwarf_cu.SetUserData(cu_sp.get()); @@ -775,15 +819,13 @@ lldb::CompUnitSP SymbolFileDWARF::ParseCompileUnit(DWARFCompileUnit &dwarf_cu) { // file is also the name of the compile unit. This // allows us to avoid loading the non-skeleton unit, // which may be in a separate DWO file. - FileSpecList support_files; + SupportFileList support_files; if (!ParseSupportFiles(dwarf_cu, module_sp, support_files)) return false; if (support_files.GetSize() == 0) return false; - initialize_cu(support_files.GetFileSpecAtIndex(0), - eLanguageTypeUnknown); - cu_sp->SetSupportFiles(std::move(support_files)); + eLanguageTypeUnknown, std::move(support_files)); return true; }; @@ -1029,7 +1071,7 @@ bool SymbolFileDWARF::ForEachExternalModule( } bool SymbolFileDWARF::ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) { + SupportFileList &support_files) { std::lock_guard guard(GetModuleMutex()); DWARFUnit *dwarf_cu = GetDWARFCompileUnit(&comp_unit); if (!dwarf_cu) @@ -1038,13 +1080,12 @@ bool SymbolFileDWARF::ParseSupportFiles(CompileUnit &comp_unit, if (!ParseSupportFiles(*dwarf_cu, comp_unit.GetModule(), support_files)) return false; - comp_unit.SetSupportFiles(support_files); return true; } bool SymbolFileDWARF::ParseSupportFiles(DWARFUnit &dwarf_cu, const ModuleSP &module, - FileSpecList &support_files) { + SupportFileList &support_files) { dw_offset_t offset = dwarf_cu.GetLineTableOffset(); if (offset == DW_INVALID_OFFSET) @@ -1057,8 +1098,8 @@ bool SymbolFileDWARF::ParseSupportFiles(DWARFUnit &dwarf_cu, return false; std::string comp_dir = dwarf_cu.GetCompilationDirectory().GetPath(); - support_files = ParseSupportFilesFromPrologue( - module, prologue, dwarf_cu.GetPathStyle(), comp_dir); + ParseSupportFilesFromPrologue(support_files, module, prologue, + dwarf_cu.GetPathStyle(), comp_dir); return true; } @@ -1070,24 +1111,27 @@ FileSpec SymbolFileDWARF::GetFile(DWARFUnit &unit, size_t file_idx) { } auto &tu = llvm::cast(unit); - return GetTypeUnitSupportFiles(tu).GetFileSpecAtIndex(file_idx); + if (const SupportFileList *support_files = GetTypeUnitSupportFiles(tu)) + return support_files->GetFileSpecAtIndex(file_idx); + return {}; } -const FileSpecList & +const SupportFileList * SymbolFileDWARF::GetTypeUnitSupportFiles(DWARFTypeUnit &tu) { - static FileSpecList empty_list; + static SupportFileList empty_list; dw_offset_t offset = tu.GetLineTableOffset(); if (offset == DW_INVALID_OFFSET || offset == llvm::DenseMapInfo::getEmptyKey() || offset == llvm::DenseMapInfo::getTombstoneKey()) - return empty_list; + return nullptr; // Many type units can share a line table, so parse the support file list // once, and cache it based on the offset field. auto iter_bool = m_type_unit_support_files.try_emplace(offset); - FileSpecList &list = iter_bool.first->second; + std::unique_ptr &list = iter_bool.first->second; if (iter_bool.second) { + list = std::make_unique(); uint64_t line_table_offset = offset; llvm::DWARFDataExtractor data = m_context.getOrLoadLineData().GetAsLLVMDWARF(); @@ -1101,14 +1145,13 @@ SymbolFileDWARF::GetTypeUnitSupportFiles(DWARFTypeUnit &tu) { }; ElapsedTime elapsed(m_parse_time); llvm::Error error = prologue.parse(data, &line_table_offset, report, ctx); - if (error) { + if (error) report(std::move(error)); - } else { - list = ParseSupportFilesFromPrologue(GetObjectFile()->GetModule(), - prologue, tu.GetPathStyle()); - } + else + ParseSupportFilesFromPrologue(*list, GetObjectFile()->GetModule(), + prologue, tu.GetPathStyle()); } - return list; + return list.get(); } bool SymbolFileDWARF::ParseIsOptimized(CompileUnit &comp_unit) { diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h index 78819edd0062..26a9502f90aa 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h @@ -123,7 +123,7 @@ public: llvm::function_ref) override; bool ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) override; + SupportFileList &support_files) override; bool ParseIsOptimized(CompileUnit &comp_unit) override; @@ -396,7 +396,7 @@ protected: bool *type_is_new); bool ParseSupportFiles(DWARFUnit &dwarf_cu, const lldb::ModuleSP &module, - FileSpecList &support_files); + SupportFileList &support_files); lldb::VariableSP ParseVariableDIE(const SymbolContext &sc, const DWARFDIE &die, @@ -489,7 +489,7 @@ protected: void FindDwpSymbolFile(); - const FileSpecList &GetTypeUnitSupportFiles(DWARFTypeUnit &tu); + const SupportFileList *GetTypeUnitSupportFiles(DWARFTypeUnit &tu); void InitializeFirstCodeAddressRecursive(const SectionList §ion_list); @@ -529,7 +529,8 @@ protected: DIEToVariableSP m_die_to_variable_sp; DIEToCompilerType m_forward_decl_die_to_compiler_type; CompilerTypeToDIE m_forward_decl_compiler_type_to_die; - llvm::DenseMap m_type_unit_support_files; + llvm::DenseMap> + m_type_unit_support_files; std::vector m_lldb_cu_to_dwarf_unit; /// DWARF does not provide a good way for traditional (concatenating) linkers /// to invalidate debug info describing dead-stripped code. These linkers will diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp index e5b59460cb85..9094a5e21e69 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp @@ -725,8 +725,8 @@ bool SymbolFileDWARFDebugMap::ForEachExternalModule( return false; } -bool SymbolFileDWARFDebugMap::ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) { +bool SymbolFileDWARFDebugMap::ParseSupportFiles( + CompileUnit &comp_unit, SupportFileList &support_files) { std::lock_guard guard(GetModuleMutex()); SymbolFileDWARF *oso_dwarf = GetSymbolFile(comp_unit); if (oso_dwarf) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h index cd0a4bb6e41c..d639ee500080 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h @@ -74,7 +74,7 @@ public: llvm::function_ref) override; bool ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) override; + SupportFileList &support_files) override; bool ParseIsOptimized(CompileUnit &comp_unit) override; diff --git a/lldb/source/Plugins/SymbolFile/JSON/SymbolFileJSON.h b/lldb/source/Plugins/SymbolFile/JSON/SymbolFileJSON.h index 4dd0d65da465..3dd33b3dc82f 100644 --- a/lldb/source/Plugins/SymbolFile/JSON/SymbolFileJSON.h +++ b/lldb/source/Plugins/SymbolFile/JSON/SymbolFileJSON.h @@ -59,7 +59,7 @@ public: bool ParseDebugMacros(CompileUnit &comp_unit) override { return false; } bool ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) override { + SupportFileList &support_files) override { return false; } diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp index ad0801339936..8375010ae3de 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp +++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp @@ -1369,7 +1369,7 @@ SymbolFileNativePDB::GetFileIndex(const CompilandIndexItem &cii, } bool SymbolFileNativePDB::ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) { + SupportFileList &support_files) { std::lock_guard guard(GetModuleMutex()); PdbSymUid cu_id(comp_unit.GetID()); lldbassert(cu_id.kind() == PdbSymUidKind::Compiland); @@ -1416,7 +1416,7 @@ void SymbolFileNativePDB::ParseInlineSite(PdbCompilandSymId id, return; InlineeSourceLine inlinee_line = iter->second; - const FileSpecList &files = comp_unit->GetSupportFiles(); + const SupportFileList &files = comp_unit->GetSupportFiles(); FileSpec decl_file; llvm::Expected file_index_or_err = GetFileIndex(*cii, inlinee_line.Header->FileID); diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h index 9d0458cf7ebf..82577771f355 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h +++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h @@ -94,7 +94,7 @@ public: bool ParseDebugMacros(lldb_private::CompileUnit &comp_unit) override; bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit, - FileSpecList &support_files) override; + SupportFileList &support_files) override; size_t ParseTypes(lldb_private::CompileUnit &comp_unit) override; bool ParseImportedModules( diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp index 9e1cd8360660..b26beecc6d12 100644 --- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp +++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp @@ -365,7 +365,7 @@ bool SymbolFilePDB::ParseDebugMacros(CompileUnit &comp_unit) { } bool SymbolFilePDB::ParseSupportFiles( - CompileUnit &comp_unit, lldb_private::FileSpecList &support_files) { + CompileUnit &comp_unit, lldb_private::SupportFileList &support_files) { // In theory this is unnecessary work for us, because all of this information // is easily (and quickly) accessible from DebugInfoPDB, so caching it a diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h index 01851f1418f3..ea495c575f1f 100644 --- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h +++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h @@ -70,7 +70,7 @@ public: bool ParseDebugMacros(lldb_private::CompileUnit &comp_unit) override; bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit, - lldb_private::FileSpecList &support_files) override; + lldb_private::SupportFileList &support_files) override; size_t ParseTypes(lldb_private::CompileUnit &comp_unit) override; diff --git a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp index 6e4c6439974e..8c17017442b1 100644 --- a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp +++ b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp @@ -211,7 +211,7 @@ bool SymbolFileSymtab::ParseDebugMacros(CompileUnit &comp_unit) { } bool SymbolFileSymtab::ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) { + SupportFileList &support_files) { return false; } diff --git a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h index 1bbc4de9c942..a36311525334 100644 --- a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h +++ b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h @@ -57,7 +57,7 @@ public: bool ParseDebugMacros(lldb_private::CompileUnit &comp_unit) override; bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit, - lldb_private::FileSpecList &support_files) override; + lldb_private::SupportFileList &support_files) override; size_t ParseTypes(lldb_private::CompileUnit &comp_unit) override; diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp index c9796973940a..a6b6c8e57eec 100644 --- a/lldb/source/Symbol/CompileUnit.cpp +++ b/lldb/source/Symbol/CompileUnit.cpp @@ -28,10 +28,11 @@ CompileUnit::CompileUnit(const lldb::ModuleSP &module_sp, void *user_data, CompileUnit::CompileUnit(const lldb::ModuleSP &module_sp, void *user_data, const FileSpec &fspec, const lldb::user_id_t cu_sym_id, lldb::LanguageType language, - lldb_private::LazyBool is_optimized) + lldb_private::LazyBool is_optimized, + SupportFileList &&support_files) : ModuleChild(module_sp), UserID(cu_sym_id), m_user_data(user_data), m_language(language), m_flags(0), m_file_spec(fspec), - m_is_optimized(is_optimized) { + m_support_files(std::move(support_files)), m_is_optimized(is_optimized) { if (language != eLanguageTypeUnknown) m_flags.Set(flagsParsedLanguage); assert(module_sp); @@ -178,10 +179,6 @@ void CompileUnit::SetLineTable(LineTable *line_table) { m_line_table_up.reset(line_table); } -void CompileUnit::SetSupportFiles(FileSpecList support_files) { - m_support_files = std::move(support_files); -} - DebugMacros *CompileUnit::GetDebugMacros() { if (m_debug_macros_sp.get() == nullptr) { if (m_flags.IsClear(flagsParsedDebugMacros)) { @@ -213,7 +210,7 @@ VariableListSP CompileUnit::GetVariableList(bool can_create) { return m_variables; } -std::vector FindFileIndexes(const FileSpecList &files, +std::vector FindFileIndexes(const SupportFileList &files, const FileSpec &file) { std::vector result; uint32_t idx = -1; @@ -411,7 +408,7 @@ bool CompileUnit::ForEachExternalModule( return false; } -const FileSpecList &CompileUnit::GetSupportFiles() { +const SupportFileList &CompileUnit::GetSupportFiles() { if (m_support_files.GetSize() == 0) { if (m_flags.IsClear(flagsParsedSupportFiles)) { m_flags.Set(flagsParsedSupportFiles); diff --git a/lldb/source/Symbol/SymbolFileOnDemand.cpp b/lldb/source/Symbol/SymbolFileOnDemand.cpp index 33995252bfe2..bdb1951d5125 100644 --- a/lldb/source/Symbol/SymbolFileOnDemand.cpp +++ b/lldb/source/Symbol/SymbolFileOnDemand.cpp @@ -115,7 +115,7 @@ bool SymbolFileOnDemand::ForEachExternalModule( } bool SymbolFileOnDemand::ParseSupportFiles(CompileUnit &comp_unit, - FileSpecList &support_files) { + SupportFileList &support_files) { LLDB_LOG(GetLog(), "[{0}] {1} is not skipped: explicitly allowed to support breakpoint", GetSymbolFileName(), __FUNCTION__); diff --git a/lldb/source/Utility/FileSpecList.cpp b/lldb/source/Utility/FileSpecList.cpp index e3d8ea650c75..8d2cf81efe5b 100644 --- a/lldb/source/Utility/FileSpecList.cpp +++ b/lldb/source/Utility/FileSpecList.cpp @@ -37,6 +37,19 @@ bool FileSpecList::AppendIfUnique(const FileSpec &file_spec) { return false; } +// FIXME: Replace this with a DenseSet at the call site. It is inefficient. +bool SupportFileList::AppendIfUnique(const FileSpec &file_spec) { + collection::iterator end = m_files.end(); + if (find_if(m_files.begin(), end, + [&](const std::unique_ptr &support_file) { + return support_file->GetSpecOnly() == file_spec; + }) == end) { + Append(file_spec); + return true; + } + return false; +} + // Clears the file list. void FileSpecList::Clear() { m_files.clear(); } @@ -55,22 +68,22 @@ void FileSpecList::Dump(Stream *s, const char *separator_cstr) const { // // Returns the valid index of the file that matches "file_spec" if it is found, // else std::numeric_limits::max() is returned. -size_t FileSpecList::FindFileIndex(size_t start_idx, const FileSpec &file_spec, - bool full) const { - const size_t num_files = m_files.size(); - +static size_t FindFileIndex(size_t start_idx, const FileSpec &file_spec, + bool full, size_t num_files, + std::function get_ith) { // When looking for files, we will compare only the filename if the FILE_SPEC // argument is empty bool compare_filename_only = file_spec.GetDirectory().IsEmpty(); for (size_t idx = start_idx; idx < num_files; ++idx) { + const FileSpec &ith = get_ith(idx); if (compare_filename_only) { - if (ConstString::Equals( - m_files[idx].GetFilename(), file_spec.GetFilename(), - file_spec.IsCaseSensitive() || m_files[idx].IsCaseSensitive())) + if (ConstString::Equals(ith.GetFilename(), file_spec.GetFilename(), + file_spec.IsCaseSensitive() || + ith.IsCaseSensitive())) return idx; } else { - if (FileSpec::Equal(m_files[idx], file_spec, full)) + if (FileSpec::Equal(ith, file_spec, full)) return idx; } } @@ -79,8 +92,24 @@ size_t FileSpecList::FindFileIndex(size_t start_idx, const FileSpec &file_spec, return UINT32_MAX; } -size_t FileSpecList::FindCompatibleIndex(size_t start_idx, - const FileSpec &file_spec) const { +size_t FileSpecList::FindFileIndex(size_t start_idx, const FileSpec &file_spec, + bool full) const { + return ::FindFileIndex( + start_idx, file_spec, full, m_files.size(), + [&](size_t idx) -> const FileSpec & { return m_files[idx]; }); +} + +size_t SupportFileList::FindFileIndex(size_t start_idx, + const FileSpec &file_spec, + bool full) const { + return ::FindFileIndex(start_idx, file_spec, full, m_files.size(), + [&](size_t idx) -> const FileSpec & { + return m_files[idx]->GetSpecOnly(); + }); +} + +size_t SupportFileList::FindCompatibleIndex(size_t start_idx, + const FileSpec &file_spec) const { const size_t num_files = m_files.size(); if (start_idx >= num_files) return UINT32_MAX; @@ -92,7 +121,7 @@ size_t FileSpecList::FindCompatibleIndex(size_t start_idx, const bool full = !file_spec.GetDirectory().IsEmpty(); for (size_t idx = start_idx; idx < num_files; ++idx) { - const FileSpec &curr_file = m_files[idx]; + const FileSpec &curr_file = m_files[idx]->GetSpecOnly(); // Always start by matching the filename first if (!curr_file.FileEquals(file_spec)) @@ -140,6 +169,13 @@ const FileSpec &FileSpecList::GetFileSpecAtIndex(size_t idx) const { return g_empty_file_spec; } +const FileSpec &SupportFileList::GetFileSpecAtIndex(size_t idx) const { + if (idx < m_files.size()) + return m_files[idx]->Materialize(); + static FileSpec g_empty_file_spec; + return g_empty_file_spec; +} + // Return the size in bytes that this object takes in memory. This returns the // size in bytes of this object's member variables and any FileSpec objects its // member variables contain, the result doesn't not include the string values diff --git a/lldb/test/API/functionalities/inline-sourcefile/Makefile b/lldb/test/API/functionalities/inline-sourcefile/Makefile new file mode 100644 index 000000000000..adb29d3a88e2 --- /dev/null +++ b/lldb/test/API/functionalities/inline-sourcefile/Makefile @@ -0,0 +1,11 @@ +C_SOURCES := main.c +CFLAGS_EXTRAS := -gdwarf-5 + +include Makefile.rules + +OBJECTS += inline.o + +$(EXE): main.c inline.o + +%.o: %.ll + $(CC) $< -c -o $@ diff --git a/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py b/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py new file mode 100644 index 000000000000..20ed0ce00661 --- /dev/null +++ b/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py @@ -0,0 +1,15 @@ +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbplatform +from lldbsuite.test import lldbutil + + +class InlineSourceFilesTestCase(TestBase): + @skipIf(compiler="gcc") + @skipIf(compiler="clang", compiler_version=["<", "18.0"]) + def test(self): + """Test DWARF inline source files.""" + self.build() + lldbutil.run_to_name_breakpoint(self, 'f') + self.expect("list f", substrs=["This is inline source code"]) diff --git a/lldb/test/API/functionalities/inline-sourcefile/inline.ll b/lldb/test/API/functionalities/inline-sourcefile/inline.ll new file mode 100644 index 000000000000..56194e45b813 --- /dev/null +++ b/lldb/test/API/functionalities/inline-sourcefile/inline.ll @@ -0,0 +1,39 @@ +; ModuleID = '/tmp/t.c' +source_filename = "/tmp/t.c" +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" + +; Function Attrs: noinline nounwind optnone ssp uwtable(sync) +define void @f() #0 !dbg !9 { +entry: + call void @stop(), !dbg !13 + ret void, !dbg !14 +} + +declare void @stop(...) #1 + +attributes #0 = { noinline nounwind optnone ssp uwtable(sync) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3, !4, !5, !6, !7} +!llvm.ident = !{!8} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0git (git@github.com:llvm/llvm-project.git 29ee66f4a0967e43a035f147c960743c7b640f2f)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: Apple, sysroot: "/") +!1 = !DIFile(filename: "/INLINE/inlined.c", directory: "/Volumes/Data/llvm-project", checksumkind: CSK_MD5, checksum: "3183154a5cb31debe9a8e27ca500bc3c") +!2 = !{i32 7, !"Dwarf Version", i32 5} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !{i32 1, !"wchar_size", i32 4} +!5 = !{i32 8, !"PIC Level", i32 2} +!6 = !{i32 7, !"uwtable", i32 1} +!7 = !{i32 7, !"frame-pointer", i32 1} +!8 = !{!"clang version 18.0.0git (git@github.com:llvm/llvm-project.git 29ee66f4a0967e43a035f147c960743c7b640f2f)"} +!9 = distinct !DISubprogram(name: "f", scope: !10, file: !10, line: 2, type: !11, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0) +!10 = !DIFile(filename: "/INLINE/inlined.c", directory: "", source: "void stop(); +void f() { + // This is inline source code. + stop(); // break here +} +") +!11 = !DISubroutineType(types: !12) +!12 = !{null} +!13 = !DILocation(line: 4, column: 3, scope: !9) +!14 = !DILocation(line: 5, column: 1, scope: !9) diff --git a/lldb/test/API/functionalities/inline-sourcefile/main.c b/lldb/test/API/functionalities/inline-sourcefile/main.c new file mode 100644 index 000000000000..c030d7773fa7 --- /dev/null +++ b/lldb/test/API/functionalities/inline-sourcefile/main.c @@ -0,0 +1,7 @@ +void f(); +void stop() {} + +int main(int argc, char const *argv[]) { + f(); + return 0; +} diff --git a/lldb/unittests/Core/FileSpecListTest.cpp b/lldb/unittests/Core/FileSpecListTest.cpp index d65e7cd2d058..e63f4a00bc3a 100644 --- a/lldb/unittests/Core/FileSpecListTest.cpp +++ b/lldb/unittests/Core/FileSpecListTest.cpp @@ -20,7 +20,7 @@ static FileSpec WindowsSpec(llvm::StringRef path) { return FileSpec(path, FileSpec::Style::windows); } -TEST(FileSpecListTest, RelativePathMatchesPosix) { +TEST(SupportFileListTest, RelativePathMatchesPosix) { const FileSpec fullpath = PosixSpec("/build/src/main.cpp"); const FileSpec relative = PosixSpec("./src/main.cpp"); @@ -32,7 +32,7 @@ TEST(FileSpecListTest, RelativePathMatchesPosix) { const FileSpec rel2_wrong = PosixSpec("asrc/main.cpp"); const FileSpec rel3_wrong = PosixSpec("rc/main.cpp"); - FileSpecList files; + SupportFileList files; files.Append(fullpath); files.Append(relative); files.Append(basename); @@ -72,7 +72,7 @@ TEST(FileSpecListTest, RelativePathMatchesPosix) { EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong)); } -TEST(FileSpecListTest, RelativePathMatchesWindows) { +TEST(SupportFileListTest, RelativePathMatchesWindows) { const FileSpec fullpath = WindowsSpec(R"(C:\build\src\main.cpp)"); const FileSpec relative = WindowsSpec(R"(.\src\main.cpp)"); @@ -84,7 +84,7 @@ TEST(FileSpecListTest, RelativePathMatchesWindows) { const FileSpec rel2_wrong = WindowsSpec(R"(asrc\main.cpp)"); const FileSpec rel3_wrong = WindowsSpec(R"("rc\main.cpp)"); - FileSpecList files; + SupportFileList files; files.Append(fullpath); files.Append(relative); files.Append(basename); -- GitLab From c12a9fc2798cc74a6ea6a8c71826047f47c815ea Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 4 Jan 2024 11:14:46 -0600 Subject: [PATCH 009/728] [ELF] Correctly set the `nvptx` triple from `makeTriple()` (#76970) Summary: The ELFObject file should be able to handle `nvptx` objects but we currently list them as unknown. This patch should now make it return `nvptx64--` correctly. --- llvm/include/llvm/Object/ELFObjectFile.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index 99477644de4d..da78e11b678d 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -1349,6 +1349,13 @@ template Triple::ArchType ELFObjectFile::getArch() const { return Triple::UnknownArch; } + case ELF::EM_CUDA: { + if (EF.getHeader().e_ident[ELF::EI_CLASS] == ELF::ELFCLASS32) + return Triple::nvptx; + else + return Triple::nvptx64; + } + case ELF::EM_BPF: return IsLittleEndian ? Triple::bpfel : Triple::bpfeb; -- GitLab From 5cd3cf107286d56cf162346d1bbbbfcc20439320 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 4 Jan 2024 16:43:35 +0000 Subject: [PATCH 010/728] [X86] cvtv2f32.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/cvtv2f32.ll | 68 +++++++++++++++---------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/llvm/test/CodeGen/X86/cvtv2f32.ll b/llvm/test/CodeGen/X86/cvtv2f32.ll index a44f16509712..b4b63caf4e91 100644 --- a/llvm/test/CodeGen/X86/cvtv2f32.ll +++ b/llvm/test/CodeGen/X86/cvtv2f32.ll @@ -1,26 +1,26 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=i686-linux-pc -mcpu=corei7 | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -mtriple=i686-linux-pc -mcpu=corei7 | FileCheck %s --check-prefix=X86 ; RUN: llc < %s -mtriple=x86_64-linux-pc -mcpu=corei7 | FileCheck %s --check-prefix=X64 ; uitofp <2 x i32> codegen from buildvector or legalization is different but gives the same results ; across the full 0 - 0xFFFFFFFF u32 range. define <2 x float> @uitofp_2i32_cvt_buildvector(i32 %x, i32 %y, <2 x float> %v) { -; X32-LABEL: uitofp_2i32_cvt_buildvector: -; X32: # %bb.0: -; X32-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero -; X32-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero -; X32-NEXT: orpd %xmm2, %xmm1 -; X32-NEXT: subsd %xmm2, %xmm1 -; X32-NEXT: cvtsd2ss %xmm1, %xmm1 -; X32-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero -; X32-NEXT: orpd %xmm2, %xmm3 -; X32-NEXT: subsd %xmm2, %xmm3 -; X32-NEXT: xorps %xmm2, %xmm2 -; X32-NEXT: cvtsd2ss %xmm3, %xmm2 -; X32-NEXT: insertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3] -; X32-NEXT: mulps %xmm1, %xmm0 -; X32-NEXT: retl +; X86-LABEL: uitofp_2i32_cvt_buildvector: +; X86: # %bb.0: +; X86-NEXT: movss {{.*#+}} xmm1 = mem[0],zero,zero,zero +; X86-NEXT: movsd {{.*#+}} xmm2 = mem[0],zero +; X86-NEXT: orpd %xmm2, %xmm1 +; X86-NEXT: subsd %xmm2, %xmm1 +; X86-NEXT: cvtsd2ss %xmm1, %xmm1 +; X86-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero +; X86-NEXT: orpd %xmm2, %xmm3 +; X86-NEXT: subsd %xmm2, %xmm3 +; X86-NEXT: xorps %xmm2, %xmm2 +; X86-NEXT: cvtsd2ss %xmm3, %xmm2 +; X86-NEXT: insertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3] +; X86-NEXT: mulps %xmm1, %xmm0 +; X86-NEXT: retl ; ; X64-LABEL: uitofp_2i32_cvt_buildvector: ; X64: # %bb.0: @@ -40,15 +40,15 @@ define <2 x float> @uitofp_2i32_cvt_buildvector(i32 %x, i32 %y, <2 x float> %v) } define <2 x float> @uitofp_2i32_buildvector_cvt(i32 %x, i32 %y, <2 x float> %v) { -; X32-LABEL: uitofp_2i32_buildvector_cvt: -; X32: # %bb.0: -; X32-NEXT: movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15] -; X32-NEXT: pmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero -; X32-NEXT: por %xmm1, %xmm2 -; X32-NEXT: subpd %xmm1, %xmm2 -; X32-NEXT: cvtpd2ps %xmm2, %xmm1 -; X32-NEXT: mulps %xmm1, %xmm0 -; X32-NEXT: retl +; X86-LABEL: uitofp_2i32_buildvector_cvt: +; X86: # %bb.0: +; X86-NEXT: movdqa {{.*#+}} xmm1 = [4.503599627370496E+15,4.503599627370496E+15] +; X86-NEXT: pmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero +; X86-NEXT: por %xmm1, %xmm2 +; X86-NEXT: subpd %xmm1, %xmm2 +; X86-NEXT: cvtpd2ps %xmm2, %xmm1 +; X86-NEXT: mulps %xmm1, %xmm0 +; X86-NEXT: retl ; ; X64-LABEL: uitofp_2i32_buildvector_cvt: ; X64: # %bb.0: @@ -69,15 +69,15 @@ define <2 x float> @uitofp_2i32_buildvector_cvt(i32 %x, i32 %y, <2 x float> %v) } define <2 x float> @uitofp_2i32_legalized(<2 x i32> %in, <2 x float> %v) { -; X32-LABEL: uitofp_2i32_legalized: -; X32: # %bb.0: -; X32-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero -; X32-NEXT: movdqa {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15] -; X32-NEXT: por %xmm2, %xmm0 -; X32-NEXT: subpd %xmm2, %xmm0 -; X32-NEXT: cvtpd2ps %xmm0, %xmm0 -; X32-NEXT: mulps %xmm1, %xmm0 -; X32-NEXT: retl +; X86-LABEL: uitofp_2i32_legalized: +; X86: # %bb.0: +; X86-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; X86-NEXT: movdqa {{.*#+}} xmm2 = [4.503599627370496E+15,4.503599627370496E+15] +; X86-NEXT: por %xmm2, %xmm0 +; X86-NEXT: subpd %xmm2, %xmm0 +; X86-NEXT: cvtpd2ps %xmm0, %xmm0 +; X86-NEXT: mulps %xmm1, %xmm0 +; X86-NEXT: retl ; ; X64-LABEL: uitofp_2i32_legalized: ; X64: # %bb.0: -- GitLab From 076dbc02724681c7d3664959d5ae742099b7edb6 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 4 Jan 2024 16:54:38 +0000 Subject: [PATCH 011/728] [X86] SimplifyDemandedVectorEltsForTargetNode - add X86ISD::VZEXT_LOAD handling. Simplify to a scalar_to_vector(load()) if we don't demand any of the upper vector elements. --- llvm/lib/Target/X86/X86ISelLowering.cpp | 14 ++++++++++++++ llvm/test/CodeGen/X86/buildvec-insertvec.ll | 6 ++---- llvm/test/CodeGen/X86/fminimum-fmaximum.ll | 5 ++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index e0679f5f27d8..fe3ba2ae2917 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -41348,6 +41348,20 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( return TLO.CombineTo(Op, Src); break; } + case X86ISD::VZEXT_LOAD: { + // If upper demanded elements are not demanded then simplify to a + // scalar_to_vector(load()). + MVT SVT = VT.getSimpleVT().getVectorElementType(); + if (DemandedElts == 1 && Op.getValue(1).use_empty() && isTypeLegal(SVT)) { + SDLoc DL(Op); + auto *Mem = cast(Op); + SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(), + Mem->getMemOperand()); + SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt); + return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Vec)); + } + break; + } case X86ISD::VBROADCAST: { SDValue Src = Op.getOperand(0); MVT SrcVT = Src.getSimpleValueType(); diff --git a/llvm/test/CodeGen/X86/buildvec-insertvec.ll b/llvm/test/CodeGen/X86/buildvec-insertvec.ll index a3568716edd9..3fdfde8576f7 100644 --- a/llvm/test/CodeGen/X86/buildvec-insertvec.ll +++ b/llvm/test/CodeGen/X86/buildvec-insertvec.ll @@ -799,9 +799,8 @@ define i32 @PR46586(ptr %p, <4 x i32> %v) { ; ; SSE41-LABEL: PR46586: ; SSE41: # %bb.0: -; SSE41-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; SSE41-NEXT: movzbl 3(%rdi), %eax ; SSE41-NEXT: extractps $3, %xmm0, %ecx -; SSE41-NEXT: pextrb $3, %xmm1, %eax ; SSE41-NEXT: xorl %edx, %edx ; SSE41-NEXT: divl %ecx ; SSE41-NEXT: movl %edx, %eax @@ -809,9 +808,8 @@ define i32 @PR46586(ptr %p, <4 x i32> %v) { ; ; AVX-LABEL: PR46586: ; AVX: # %bb.0: -; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero +; AVX-NEXT: movzbl 3(%rdi), %eax ; AVX-NEXT: vextractps $3, %xmm0, %ecx -; AVX-NEXT: vpextrb $3, %xmm1, %eax ; AVX-NEXT: xorl %edx, %edx ; AVX-NEXT: divl %ecx ; AVX-NEXT: movl %edx, %eax diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll index 5bb5d1e9c17e..8905d2bce5e9 100644 --- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll +++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll @@ -699,10 +699,9 @@ define double @test_fminimum_nnan(double %x, double %y) "no-nans-fp-math"="true" ; X86-NEXT: movl %esp, %ebp ; X86-NEXT: andl $-8, %esp ; X86-NEXT: subl $8, %esp -; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; X86-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero -; X86-NEXT: vmovsd {{.*#+}} xmm2 = mem[0],zero -; X86-NEXT: vextractps $1, %xmm2, %eax +; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; X86-NEXT: vextractps $1, %xmm0, %eax ; X86-NEXT: testl %eax, %eax ; X86-NEXT: js .LBB14_1 ; X86-NEXT: # %bb.2: -- GitLab From ce4459d5903fe53065f4c198cd71be6e514475c2 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 4 Jan 2024 17:11:35 +0000 Subject: [PATCH 012/728] [X86] 64-bit-shift-by-32-minus-y.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- .../CodeGen/X86/64-bit-shift-by-32-minus-y.ll | 550 +++++++++--------- 1 file changed, 275 insertions(+), 275 deletions(-) diff --git a/llvm/test/CodeGen/X86/64-bit-shift-by-32-minus-y.ll b/llvm/test/CodeGen/X86/64-bit-shift-by-32-minus-y.ll index 705a1cc861e0..4c92adb25d0b 100644 --- a/llvm/test/CodeGen/X86/64-bit-shift-by-32-minus-y.ll +++ b/llvm/test/CodeGen/X86/64-bit-shift-by-32-minus-y.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64-NOBMI2 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi2 | FileCheck %s --check-prefixes=X64-BMI2 -; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X32-NOBMI2 -; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+bmi2 | FileCheck %s --check-prefixes=X32-BMI2 +; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86-NOBMI2 +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+bmi2 | FileCheck %s --check-prefixes=X86-BMI2 define i64 @t0(i64 %val, i64 %shamt) nounwind { ; X64-NOBMI2-LABEL: t0: @@ -21,40 +21,40 @@ define i64 @t0(i64 %val, i64 %shamt) nounwind { ; X64-BMI2-NEXT: shlxq %rsi, %rdi, %rax ; X64-BMI2-NEXT: retq ; -; X32-NOBMI2-LABEL: t0: -; X32-NOBMI2: # %bb.0: -; X32-NOBMI2-NEXT: pushl %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NOBMI2-NEXT: movb $32, %cl -; X32-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-NOBMI2-NEXT: movl %esi, %eax -; X32-NOBMI2-NEXT: shll %cl, %eax -; X32-NOBMI2-NEXT: shldl %cl, %esi, %edx -; X32-NOBMI2-NEXT: testb $32, %cl -; X32-NOBMI2-NEXT: je .LBB0_2 -; X32-NOBMI2-NEXT: # %bb.1: -; X32-NOBMI2-NEXT: movl %eax, %edx -; X32-NOBMI2-NEXT: xorl %eax, %eax -; X32-NOBMI2-NEXT: .LBB0_2: -; X32-NOBMI2-NEXT: popl %esi -; X32-NOBMI2-NEXT: retl +; X86-NOBMI2-LABEL: t0: +; X86-NOBMI2: # %bb.0: +; X86-NOBMI2-NEXT: pushl %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NOBMI2-NEXT: movb $32, %cl +; X86-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-NOBMI2-NEXT: movl %esi, %eax +; X86-NOBMI2-NEXT: shll %cl, %eax +; X86-NOBMI2-NEXT: shldl %cl, %esi, %edx +; X86-NOBMI2-NEXT: testb $32, %cl +; X86-NOBMI2-NEXT: je .LBB0_2 +; X86-NOBMI2-NEXT: # %bb.1: +; X86-NOBMI2-NEXT: movl %eax, %edx +; X86-NOBMI2-NEXT: xorl %eax, %eax +; X86-NOBMI2-NEXT: .LBB0_2: +; X86-NOBMI2-NEXT: popl %esi +; X86-NOBMI2-NEXT: retl ; -; X32-BMI2-LABEL: t0: -; X32-BMI2: # %bb.0: -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-BMI2-NEXT: movb $32, %cl -; X32-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-BMI2-NEXT: shldl %cl, %eax, %edx -; X32-BMI2-NEXT: shlxl %ecx, %eax, %eax -; X32-BMI2-NEXT: testb $32, %cl -; X32-BMI2-NEXT: je .LBB0_2 -; X32-BMI2-NEXT: # %bb.1: -; X32-BMI2-NEXT: movl %eax, %edx -; X32-BMI2-NEXT: xorl %eax, %eax -; X32-BMI2-NEXT: .LBB0_2: -; X32-BMI2-NEXT: retl +; X86-BMI2-LABEL: t0: +; X86-BMI2: # %bb.0: +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-BMI2-NEXT: movb $32, %cl +; X86-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-BMI2-NEXT: shldl %cl, %eax, %edx +; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax +; X86-BMI2-NEXT: testb $32, %cl +; X86-BMI2-NEXT: je .LBB0_2 +; X86-BMI2-NEXT: # %bb.1: +; X86-BMI2-NEXT: movl %eax, %edx +; X86-BMI2-NEXT: xorl %eax, %eax +; X86-BMI2-NEXT: .LBB0_2: +; X86-BMI2-NEXT: retl %negshamt = sub i64 32, %shamt %shifted = shl i64 %val, %negshamt ret i64 %shifted @@ -77,40 +77,40 @@ define i64 @n1(i64 %val, i64 %shamt) nounwind { ; X64-BMI2-NEXT: shlxq %rax, %rdi, %rax ; X64-BMI2-NEXT: retq ; -; X32-NOBMI2-LABEL: n1: -; X32-NOBMI2: # %bb.0: -; X32-NOBMI2-NEXT: pushl %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NOBMI2-NEXT: movb $33, %cl -; X32-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-NOBMI2-NEXT: movl %esi, %eax -; X32-NOBMI2-NEXT: shll %cl, %eax -; X32-NOBMI2-NEXT: shldl %cl, %esi, %edx -; X32-NOBMI2-NEXT: testb $32, %cl -; X32-NOBMI2-NEXT: je .LBB1_2 -; X32-NOBMI2-NEXT: # %bb.1: -; X32-NOBMI2-NEXT: movl %eax, %edx -; X32-NOBMI2-NEXT: xorl %eax, %eax -; X32-NOBMI2-NEXT: .LBB1_2: -; X32-NOBMI2-NEXT: popl %esi -; X32-NOBMI2-NEXT: retl +; X86-NOBMI2-LABEL: n1: +; X86-NOBMI2: # %bb.0: +; X86-NOBMI2-NEXT: pushl %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NOBMI2-NEXT: movb $33, %cl +; X86-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-NOBMI2-NEXT: movl %esi, %eax +; X86-NOBMI2-NEXT: shll %cl, %eax +; X86-NOBMI2-NEXT: shldl %cl, %esi, %edx +; X86-NOBMI2-NEXT: testb $32, %cl +; X86-NOBMI2-NEXT: je .LBB1_2 +; X86-NOBMI2-NEXT: # %bb.1: +; X86-NOBMI2-NEXT: movl %eax, %edx +; X86-NOBMI2-NEXT: xorl %eax, %eax +; X86-NOBMI2-NEXT: .LBB1_2: +; X86-NOBMI2-NEXT: popl %esi +; X86-NOBMI2-NEXT: retl ; -; X32-BMI2-LABEL: n1: -; X32-BMI2: # %bb.0: -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-BMI2-NEXT: movb $33, %cl -; X32-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-BMI2-NEXT: shldl %cl, %eax, %edx -; X32-BMI2-NEXT: shlxl %ecx, %eax, %eax -; X32-BMI2-NEXT: testb $32, %cl -; X32-BMI2-NEXT: je .LBB1_2 -; X32-BMI2-NEXT: # %bb.1: -; X32-BMI2-NEXT: movl %eax, %edx -; X32-BMI2-NEXT: xorl %eax, %eax -; X32-BMI2-NEXT: .LBB1_2: -; X32-BMI2-NEXT: retl +; X86-BMI2-LABEL: n1: +; X86-BMI2: # %bb.0: +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-BMI2-NEXT: movb $33, %cl +; X86-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-BMI2-NEXT: shldl %cl, %eax, %edx +; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax +; X86-BMI2-NEXT: testb $32, %cl +; X86-BMI2-NEXT: je .LBB1_2 +; X86-BMI2-NEXT: # %bb.1: +; X86-BMI2-NEXT: movl %eax, %edx +; X86-BMI2-NEXT: xorl %eax, %eax +; X86-BMI2-NEXT: .LBB1_2: +; X86-BMI2-NEXT: retl %negshamt = sub i64 33, %shamt %shifted = shl i64 %val, %negshamt ret i64 %shifted @@ -131,40 +131,40 @@ define i64 @n2(i64 %val, i64 %shamt) nounwind { ; X64-BMI2-NEXT: shlxq %rax, %rdi, %rax ; X64-BMI2-NEXT: retq ; -; X32-NOBMI2-LABEL: n2: -; X32-NOBMI2: # %bb.0: -; X32-NOBMI2-NEXT: pushl %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NOBMI2-NEXT: movb $31, %cl -; X32-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-NOBMI2-NEXT: movl %esi, %eax -; X32-NOBMI2-NEXT: shll %cl, %eax -; X32-NOBMI2-NEXT: shldl %cl, %esi, %edx -; X32-NOBMI2-NEXT: testb $32, %cl -; X32-NOBMI2-NEXT: je .LBB2_2 -; X32-NOBMI2-NEXT: # %bb.1: -; X32-NOBMI2-NEXT: movl %eax, %edx -; X32-NOBMI2-NEXT: xorl %eax, %eax -; X32-NOBMI2-NEXT: .LBB2_2: -; X32-NOBMI2-NEXT: popl %esi -; X32-NOBMI2-NEXT: retl +; X86-NOBMI2-LABEL: n2: +; X86-NOBMI2: # %bb.0: +; X86-NOBMI2-NEXT: pushl %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NOBMI2-NEXT: movb $31, %cl +; X86-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-NOBMI2-NEXT: movl %esi, %eax +; X86-NOBMI2-NEXT: shll %cl, %eax +; X86-NOBMI2-NEXT: shldl %cl, %esi, %edx +; X86-NOBMI2-NEXT: testb $32, %cl +; X86-NOBMI2-NEXT: je .LBB2_2 +; X86-NOBMI2-NEXT: # %bb.1: +; X86-NOBMI2-NEXT: movl %eax, %edx +; X86-NOBMI2-NEXT: xorl %eax, %eax +; X86-NOBMI2-NEXT: .LBB2_2: +; X86-NOBMI2-NEXT: popl %esi +; X86-NOBMI2-NEXT: retl ; -; X32-BMI2-LABEL: n2: -; X32-BMI2: # %bb.0: -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-BMI2-NEXT: movb $31, %cl -; X32-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-BMI2-NEXT: shldl %cl, %eax, %edx -; X32-BMI2-NEXT: shlxl %ecx, %eax, %eax -; X32-BMI2-NEXT: testb $32, %cl -; X32-BMI2-NEXT: je .LBB2_2 -; X32-BMI2-NEXT: # %bb.1: -; X32-BMI2-NEXT: movl %eax, %edx -; X32-BMI2-NEXT: xorl %eax, %eax -; X32-BMI2-NEXT: .LBB2_2: -; X32-BMI2-NEXT: retl +; X86-BMI2-LABEL: n2: +; X86-BMI2: # %bb.0: +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-BMI2-NEXT: movb $31, %cl +; X86-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-BMI2-NEXT: shldl %cl, %eax, %edx +; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax +; X86-BMI2-NEXT: testb $32, %cl +; X86-BMI2-NEXT: je .LBB2_2 +; X86-BMI2-NEXT: # %bb.1: +; X86-BMI2-NEXT: movl %eax, %edx +; X86-BMI2-NEXT: xorl %eax, %eax +; X86-BMI2-NEXT: .LBB2_2: +; X86-BMI2-NEXT: retl %negshamt = sub i64 31, %shamt %shifted = shl i64 %val, %negshamt ret i64 %shifted @@ -186,40 +186,40 @@ define i64 @t3(i64 %val, i64 %shamt) nounwind { ; X64-BMI2-NEXT: shlxq %rsi, %rdi, %rax ; X64-BMI2-NEXT: retq ; -; X32-NOBMI2-LABEL: t3: -; X32-NOBMI2: # %bb.0: -; X32-NOBMI2-NEXT: pushl %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NOBMI2-NEXT: movb $64, %cl -; X32-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-NOBMI2-NEXT: movl %esi, %eax -; X32-NOBMI2-NEXT: shll %cl, %eax -; X32-NOBMI2-NEXT: shldl %cl, %esi, %edx -; X32-NOBMI2-NEXT: testb $32, %cl -; X32-NOBMI2-NEXT: je .LBB3_2 -; X32-NOBMI2-NEXT: # %bb.1: -; X32-NOBMI2-NEXT: movl %eax, %edx -; X32-NOBMI2-NEXT: xorl %eax, %eax -; X32-NOBMI2-NEXT: .LBB3_2: -; X32-NOBMI2-NEXT: popl %esi -; X32-NOBMI2-NEXT: retl +; X86-NOBMI2-LABEL: t3: +; X86-NOBMI2: # %bb.0: +; X86-NOBMI2-NEXT: pushl %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NOBMI2-NEXT: movb $64, %cl +; X86-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-NOBMI2-NEXT: movl %esi, %eax +; X86-NOBMI2-NEXT: shll %cl, %eax +; X86-NOBMI2-NEXT: shldl %cl, %esi, %edx +; X86-NOBMI2-NEXT: testb $32, %cl +; X86-NOBMI2-NEXT: je .LBB3_2 +; X86-NOBMI2-NEXT: # %bb.1: +; X86-NOBMI2-NEXT: movl %eax, %edx +; X86-NOBMI2-NEXT: xorl %eax, %eax +; X86-NOBMI2-NEXT: .LBB3_2: +; X86-NOBMI2-NEXT: popl %esi +; X86-NOBMI2-NEXT: retl ; -; X32-BMI2-LABEL: t3: -; X32-BMI2: # %bb.0: -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-BMI2-NEXT: movb $64, %cl -; X32-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-BMI2-NEXT: shldl %cl, %eax, %edx -; X32-BMI2-NEXT: shlxl %ecx, %eax, %eax -; X32-BMI2-NEXT: testb $32, %cl -; X32-BMI2-NEXT: je .LBB3_2 -; X32-BMI2-NEXT: # %bb.1: -; X32-BMI2-NEXT: movl %eax, %edx -; X32-BMI2-NEXT: xorl %eax, %eax -; X32-BMI2-NEXT: .LBB3_2: -; X32-BMI2-NEXT: retl +; X86-BMI2-LABEL: t3: +; X86-BMI2: # %bb.0: +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-BMI2-NEXT: movb $64, %cl +; X86-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-BMI2-NEXT: shldl %cl, %eax, %edx +; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax +; X86-BMI2-NEXT: testb $32, %cl +; X86-BMI2-NEXT: je .LBB3_2 +; X86-BMI2-NEXT: # %bb.1: +; X86-BMI2-NEXT: movl %eax, %edx +; X86-BMI2-NEXT: xorl %eax, %eax +; X86-BMI2-NEXT: .LBB3_2: +; X86-BMI2-NEXT: retl %negshamt = sub i64 64, %shamt %shifted = shl i64 %val, %negshamt ret i64 %shifted @@ -242,40 +242,40 @@ define i64 @t4(i64 %val, i64 %shamt) nounwind { ; X64-BMI2-NEXT: shlxq %rsi, %rdi, %rax ; X64-BMI2-NEXT: retq ; -; X32-NOBMI2-LABEL: t4: -; X32-NOBMI2: # %bb.0: -; X32-NOBMI2-NEXT: pushl %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NOBMI2-NEXT: movb $96, %cl -; X32-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-NOBMI2-NEXT: movl %esi, %eax -; X32-NOBMI2-NEXT: shll %cl, %eax -; X32-NOBMI2-NEXT: shldl %cl, %esi, %edx -; X32-NOBMI2-NEXT: testb $32, %cl -; X32-NOBMI2-NEXT: je .LBB4_2 -; X32-NOBMI2-NEXT: # %bb.1: -; X32-NOBMI2-NEXT: movl %eax, %edx -; X32-NOBMI2-NEXT: xorl %eax, %eax -; X32-NOBMI2-NEXT: .LBB4_2: -; X32-NOBMI2-NEXT: popl %esi -; X32-NOBMI2-NEXT: retl +; X86-NOBMI2-LABEL: t4: +; X86-NOBMI2: # %bb.0: +; X86-NOBMI2-NEXT: pushl %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NOBMI2-NEXT: movb $96, %cl +; X86-NOBMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-NOBMI2-NEXT: movl %esi, %eax +; X86-NOBMI2-NEXT: shll %cl, %eax +; X86-NOBMI2-NEXT: shldl %cl, %esi, %edx +; X86-NOBMI2-NEXT: testb $32, %cl +; X86-NOBMI2-NEXT: je .LBB4_2 +; X86-NOBMI2-NEXT: # %bb.1: +; X86-NOBMI2-NEXT: movl %eax, %edx +; X86-NOBMI2-NEXT: xorl %eax, %eax +; X86-NOBMI2-NEXT: .LBB4_2: +; X86-NOBMI2-NEXT: popl %esi +; X86-NOBMI2-NEXT: retl ; -; X32-BMI2-LABEL: t4: -; X32-BMI2: # %bb.0: -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-BMI2-NEXT: movb $96, %cl -; X32-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl -; X32-BMI2-NEXT: shldl %cl, %eax, %edx -; X32-BMI2-NEXT: shlxl %ecx, %eax, %eax -; X32-BMI2-NEXT: testb $32, %cl -; X32-BMI2-NEXT: je .LBB4_2 -; X32-BMI2-NEXT: # %bb.1: -; X32-BMI2-NEXT: movl %eax, %edx -; X32-BMI2-NEXT: xorl %eax, %eax -; X32-BMI2-NEXT: .LBB4_2: -; X32-BMI2-NEXT: retl +; X86-BMI2-LABEL: t4: +; X86-BMI2: # %bb.0: +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-BMI2-NEXT: movb $96, %cl +; X86-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl +; X86-BMI2-NEXT: shldl %cl, %eax, %edx +; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax +; X86-BMI2-NEXT: testb $32, %cl +; X86-BMI2-NEXT: je .LBB4_2 +; X86-BMI2-NEXT: # %bb.1: +; X86-BMI2-NEXT: movl %eax, %edx +; X86-BMI2-NEXT: xorl %eax, %eax +; X86-BMI2-NEXT: .LBB4_2: +; X86-BMI2-NEXT: retl %negshamt = sub i64 96, %shamt %shifted = shl i64 %val, %negshamt ret i64 %shifted @@ -300,66 +300,66 @@ define i64 @t5_cse(i64 %val, i64 %shamt, ptr%dst) nounwind { ; X64-BMI2-NEXT: shlxq %rsi, %rdi, %rax ; X64-BMI2-NEXT: retq ; -; X32-NOBMI2-LABEL: t5_cse: -; X32-NOBMI2: # %bb.0: -; X32-NOBMI2-NEXT: pushl %ebx -; X32-NOBMI2-NEXT: pushl %edi -; X32-NOBMI2-NEXT: pushl %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edi -; X32-NOBMI2-NEXT: movl %eax, %ebx -; X32-NOBMI2-NEXT: addl $32, %ebx -; X32-NOBMI2-NEXT: adcl $0, %edi -; X32-NOBMI2-NEXT: movl %ebx, (%ecx) -; X32-NOBMI2-NEXT: movl %edi, 4(%ecx) -; X32-NOBMI2-NEXT: movb $32, %cl -; X32-NOBMI2-NEXT: subb %al, %cl -; X32-NOBMI2-NEXT: movl %esi, %eax -; X32-NOBMI2-NEXT: shll %cl, %eax -; X32-NOBMI2-NEXT: shldl %cl, %esi, %edx -; X32-NOBMI2-NEXT: testb $32, %cl -; X32-NOBMI2-NEXT: je .LBB5_2 -; X32-NOBMI2-NEXT: # %bb.1: -; X32-NOBMI2-NEXT: movl %eax, %edx -; X32-NOBMI2-NEXT: xorl %eax, %eax -; X32-NOBMI2-NEXT: .LBB5_2: -; X32-NOBMI2-NEXT: popl %esi -; X32-NOBMI2-NEXT: popl %edi -; X32-NOBMI2-NEXT: popl %ebx -; X32-NOBMI2-NEXT: retl +; X86-NOBMI2-LABEL: t5_cse: +; X86-NOBMI2: # %bb.0: +; X86-NOBMI2-NEXT: pushl %ebx +; X86-NOBMI2-NEXT: pushl %edi +; X86-NOBMI2-NEXT: pushl %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NOBMI2-NEXT: movl %eax, %ebx +; X86-NOBMI2-NEXT: addl $32, %ebx +; X86-NOBMI2-NEXT: adcl $0, %edi +; X86-NOBMI2-NEXT: movl %ebx, (%ecx) +; X86-NOBMI2-NEXT: movl %edi, 4(%ecx) +; X86-NOBMI2-NEXT: movb $32, %cl +; X86-NOBMI2-NEXT: subb %al, %cl +; X86-NOBMI2-NEXT: movl %esi, %eax +; X86-NOBMI2-NEXT: shll %cl, %eax +; X86-NOBMI2-NEXT: shldl %cl, %esi, %edx +; X86-NOBMI2-NEXT: testb $32, %cl +; X86-NOBMI2-NEXT: je .LBB5_2 +; X86-NOBMI2-NEXT: # %bb.1: +; X86-NOBMI2-NEXT: movl %eax, %edx +; X86-NOBMI2-NEXT: xorl %eax, %eax +; X86-NOBMI2-NEXT: .LBB5_2: +; X86-NOBMI2-NEXT: popl %esi +; X86-NOBMI2-NEXT: popl %edi +; X86-NOBMI2-NEXT: popl %ebx +; X86-NOBMI2-NEXT: retl ; -; X32-BMI2-LABEL: t5_cse: -; X32-BMI2: # %bb.0: -; X32-BMI2-NEXT: pushl %ebx -; X32-BMI2-NEXT: pushl %edi -; X32-BMI2-NEXT: pushl %esi -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebx -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-BMI2-NEXT: movl %ebx, %edi -; X32-BMI2-NEXT: addl $32, %edi -; X32-BMI2-NEXT: adcl $0, %esi -; X32-BMI2-NEXT: movl %edi, (%ecx) -; X32-BMI2-NEXT: movl %esi, 4(%ecx) -; X32-BMI2-NEXT: movb $32, %cl -; X32-BMI2-NEXT: subb %bl, %cl -; X32-BMI2-NEXT: shldl %cl, %eax, %edx -; X32-BMI2-NEXT: shlxl %ecx, %eax, %eax -; X32-BMI2-NEXT: testb $32, %cl -; X32-BMI2-NEXT: je .LBB5_2 -; X32-BMI2-NEXT: # %bb.1: -; X32-BMI2-NEXT: movl %eax, %edx -; X32-BMI2-NEXT: xorl %eax, %eax -; X32-BMI2-NEXT: .LBB5_2: -; X32-BMI2-NEXT: popl %esi -; X32-BMI2-NEXT: popl %edi -; X32-BMI2-NEXT: popl %ebx -; X32-BMI2-NEXT: retl +; X86-BMI2-LABEL: t5_cse: +; X86-BMI2: # %bb.0: +; X86-BMI2-NEXT: pushl %ebx +; X86-BMI2-NEXT: pushl %edi +; X86-BMI2-NEXT: pushl %esi +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-BMI2-NEXT: movl %ebx, %edi +; X86-BMI2-NEXT: addl $32, %edi +; X86-BMI2-NEXT: adcl $0, %esi +; X86-BMI2-NEXT: movl %edi, (%ecx) +; X86-BMI2-NEXT: movl %esi, 4(%ecx) +; X86-BMI2-NEXT: movb $32, %cl +; X86-BMI2-NEXT: subb %bl, %cl +; X86-BMI2-NEXT: shldl %cl, %eax, %edx +; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax +; X86-BMI2-NEXT: testb $32, %cl +; X86-BMI2-NEXT: je .LBB5_2 +; X86-BMI2-NEXT: # %bb.1: +; X86-BMI2-NEXT: movl %eax, %edx +; X86-BMI2-NEXT: xorl %eax, %eax +; X86-BMI2-NEXT: .LBB5_2: +; X86-BMI2-NEXT: popl %esi +; X86-BMI2-NEXT: popl %edi +; X86-BMI2-NEXT: popl %ebx +; X86-BMI2-NEXT: retl %incshamt = add i64 %shamt, 32 store i64 %incshamt, ptr %dst %negshamt = sub i64 32, %shamt @@ -386,56 +386,56 @@ define i64 @t6_cse2(i64 %val, i64 %shamt, ptr%dst) nounwind { ; X64-BMI2-NEXT: shlxq %rax, %rdi, %rax ; X64-BMI2-NEXT: retq ; -; X32-NOBMI2-LABEL: t6_cse2: -; X32-NOBMI2: # %bb.0: -; X32-NOBMI2-NEXT: pushl %edi -; X32-NOBMI2-NEXT: pushl %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NOBMI2-NEXT: xorl %edi, %edi -; X32-NOBMI2-NEXT: movl $32, %ecx -; X32-NOBMI2-NEXT: subl {{[0-9]+}}(%esp), %ecx -; X32-NOBMI2-NEXT: sbbl {{[0-9]+}}(%esp), %edi -; X32-NOBMI2-NEXT: movl %ecx, (%eax) -; X32-NOBMI2-NEXT: movl %edi, 4(%eax) -; X32-NOBMI2-NEXT: movl %esi, %eax -; X32-NOBMI2-NEXT: shll %cl, %eax -; X32-NOBMI2-NEXT: shldl %cl, %esi, %edx -; X32-NOBMI2-NEXT: testb $32, %cl -; X32-NOBMI2-NEXT: je .LBB6_2 -; X32-NOBMI2-NEXT: # %bb.1: -; X32-NOBMI2-NEXT: movl %eax, %edx -; X32-NOBMI2-NEXT: xorl %eax, %eax -; X32-NOBMI2-NEXT: .LBB6_2: -; X32-NOBMI2-NEXT: popl %esi -; X32-NOBMI2-NEXT: popl %edi -; X32-NOBMI2-NEXT: retl +; X86-NOBMI2-LABEL: t6_cse2: +; X86-NOBMI2: # %bb.0: +; X86-NOBMI2-NEXT: pushl %edi +; X86-NOBMI2-NEXT: pushl %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NOBMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOBMI2-NEXT: xorl %edi, %edi +; X86-NOBMI2-NEXT: movl $32, %ecx +; X86-NOBMI2-NEXT: subl {{[0-9]+}}(%esp), %ecx +; X86-NOBMI2-NEXT: sbbl {{[0-9]+}}(%esp), %edi +; X86-NOBMI2-NEXT: movl %ecx, (%eax) +; X86-NOBMI2-NEXT: movl %edi, 4(%eax) +; X86-NOBMI2-NEXT: movl %esi, %eax +; X86-NOBMI2-NEXT: shll %cl, %eax +; X86-NOBMI2-NEXT: shldl %cl, %esi, %edx +; X86-NOBMI2-NEXT: testb $32, %cl +; X86-NOBMI2-NEXT: je .LBB6_2 +; X86-NOBMI2-NEXT: # %bb.1: +; X86-NOBMI2-NEXT: movl %eax, %edx +; X86-NOBMI2-NEXT: xorl %eax, %eax +; X86-NOBMI2-NEXT: .LBB6_2: +; X86-NOBMI2-NEXT: popl %esi +; X86-NOBMI2-NEXT: popl %edi +; X86-NOBMI2-NEXT: retl ; -; X32-BMI2-LABEL: t6_cse2: -; X32-BMI2: # %bb.0: -; X32-BMI2-NEXT: pushl %edi -; X32-BMI2-NEXT: pushl %esi -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-BMI2-NEXT: xorl %edi, %edi -; X32-BMI2-NEXT: movl $32, %ecx -; X32-BMI2-NEXT: subl {{[0-9]+}}(%esp), %ecx -; X32-BMI2-NEXT: sbbl {{[0-9]+}}(%esp), %edi -; X32-BMI2-NEXT: movl %ecx, (%esi) -; X32-BMI2-NEXT: movl %edi, 4(%esi) -; X32-BMI2-NEXT: shldl %cl, %eax, %edx -; X32-BMI2-NEXT: shlxl %ecx, %eax, %eax -; X32-BMI2-NEXT: testb $32, %cl -; X32-BMI2-NEXT: je .LBB6_2 -; X32-BMI2-NEXT: # %bb.1: -; X32-BMI2-NEXT: movl %eax, %edx -; X32-BMI2-NEXT: xorl %eax, %eax -; X32-BMI2-NEXT: .LBB6_2: -; X32-BMI2-NEXT: popl %esi -; X32-BMI2-NEXT: popl %edi -; X32-BMI2-NEXT: retl +; X86-BMI2-LABEL: t6_cse2: +; X86-BMI2: # %bb.0: +; X86-BMI2-NEXT: pushl %edi +; X86-BMI2-NEXT: pushl %esi +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-BMI2-NEXT: xorl %edi, %edi +; X86-BMI2-NEXT: movl $32, %ecx +; X86-BMI2-NEXT: subl {{[0-9]+}}(%esp), %ecx +; X86-BMI2-NEXT: sbbl {{[0-9]+}}(%esp), %edi +; X86-BMI2-NEXT: movl %ecx, (%esi) +; X86-BMI2-NEXT: movl %edi, 4(%esi) +; X86-BMI2-NEXT: shldl %cl, %eax, %edx +; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax +; X86-BMI2-NEXT: testb $32, %cl +; X86-BMI2-NEXT: je .LBB6_2 +; X86-BMI2-NEXT: # %bb.1: +; X86-BMI2-NEXT: movl %eax, %edx +; X86-BMI2-NEXT: xorl %eax, %eax +; X86-BMI2-NEXT: .LBB6_2: +; X86-BMI2-NEXT: popl %esi +; X86-BMI2-NEXT: popl %edi +; X86-BMI2-NEXT: retl %negshamt = sub i64 32, %shamt store i64 %negshamt, ptr %dst %shifted = shl i64 %val, %negshamt -- GitLab From 63e30747815a20e9976c5bcedb81e8b44cbec582 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 4 Jan 2024 17:12:12 +0000 Subject: [PATCH 013/728] [X86] aligned-variadic.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/aligned-variadic.ll | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/test/CodeGen/X86/aligned-variadic.ll b/llvm/test/CodeGen/X86/aligned-variadic.ll index 28cfff9d85a6..bc60662def2b 100644 --- a/llvm/test/CodeGen/X86/aligned-variadic.ll +++ b/llvm/test/CodeGen/X86/aligned-variadic.ll @@ -1,5 +1,5 @@ ; RUN: llc < %s -mtriple=x86_64-apple-darwin -stack-symbol-ordering=0 | FileCheck %s -check-prefix=X64 -; RUN: llc < %s -mtriple=i686-apple-darwin -stack-symbol-ordering=0 | FileCheck %s -check-prefix=X32 +; RUN: llc < %s -mtriple=i686-apple-darwin -stack-symbol-ordering=0 | FileCheck %s -check-prefix=X86 %struct.Baz = type { [17 x i8] } %struct.__va_list_tag = type { i32, i32, ptr, ptr } @@ -13,8 +13,8 @@ entry: %overflow_arg_area = load ptr, ptr %overflow_arg_area_p, align 8 %overflow_arg_area.next = getelementptr i8, ptr %overflow_arg_area, i64 24 store ptr %overflow_arg_area.next, ptr %overflow_arg_area_p, align 8 -; X32: leal 68(%esp), [[REG:%.*]] -; X32: movl [[REG]], 16(%esp) +; X86: leal 68(%esp), [[REG:%.*]] +; X86: movl [[REG]], 16(%esp) ; X64: leaq 256(%rsp), [[REG:%.*]] ; X64: movq [[REG]], 184(%rsp) ; X64: leaq 176(%rsp), %rdi -- GitLab From 2cbf6526157958531e5765e7aa6faa53bfac5c5a Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 4 Jan 2024 17:13:20 +0000 Subject: [PATCH 014/728] [X86] avx512-pmovxrm.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/avx512-pmovxrm.ll | 122 ++++++++++++------------ 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/llvm/test/CodeGen/X86/avx512-pmovxrm.ll b/llvm/test/CodeGen/X86/avx512-pmovxrm.ll index 0b3190be62a5..237ae7a5c64a 100644 --- a/llvm/test/CodeGen/X86/avx512-pmovxrm.ll +++ b/llvm/test/CodeGen/X86/avx512-pmovxrm.ll @@ -1,13 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -disable-peephole -mtriple=i686-apple-darwin -mattr=+avx512f,avx512bw | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -disable-peephole -mtriple=i686-apple-darwin -mattr=+avx512f,avx512bw | FileCheck %s --check-prefix=X86 ; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+avx512f,avx512bw | FileCheck %s --check-prefix=X64 define <32 x i16> @test_llvm_x86_avx512_pmovsxbw(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovsxbw: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovsxbw (%eax), %zmm0 -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovsxbw: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovsxbw (%eax), %zmm0 +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovsxbw: ; X64: ## %bb.0: @@ -19,11 +19,11 @@ define <32 x i16> @test_llvm_x86_avx512_pmovsxbw(ptr %a) { } define <16 x i32> @test_llvm_x86_avx512_pmovsxbd(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovsxbd: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovsxbd (%eax), %zmm0 -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovsxbd: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovsxbd (%eax), %zmm0 +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovsxbd: ; X64: ## %bb.0: @@ -35,11 +35,11 @@ define <16 x i32> @test_llvm_x86_avx512_pmovsxbd(ptr %a) { } define <8 x i64> @test_llvm_x86_avx512_pmovsxbq(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovsxbq: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovsxbq (%eax), %zmm0 -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovsxbq: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovsxbq (%eax), %zmm0 +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovsxbq: ; X64: ## %bb.0: @@ -52,11 +52,11 @@ define <8 x i64> @test_llvm_x86_avx512_pmovsxbq(ptr %a) { } define <16 x i32> @test_llvm_x86_avx512_pmovsxwd(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovsxwd: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovsxwd (%eax), %zmm0 -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovsxwd: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovsxwd (%eax), %zmm0 +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovsxwd: ; X64: ## %bb.0: @@ -68,11 +68,11 @@ define <16 x i32> @test_llvm_x86_avx512_pmovsxwd(ptr %a) { } define <8 x i64> @test_llvm_x86_avx512_pmovsxwq(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovsxwq: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovsxwq (%eax), %zmm0 -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovsxwq: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovsxwq (%eax), %zmm0 +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovsxwq: ; X64: ## %bb.0: @@ -84,11 +84,11 @@ define <8 x i64> @test_llvm_x86_avx512_pmovsxwq(ptr %a) { } define <8 x i64> @test_llvm_x86_avx512_pmovsxdq(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovsxdq: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovsxdq (%eax), %zmm0 -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovsxdq: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovsxdq (%eax), %zmm0 +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovsxdq: ; X64: ## %bb.0: @@ -100,11 +100,11 @@ define <8 x i64> @test_llvm_x86_avx512_pmovsxdq(ptr %a) { } define <32 x i16> @test_llvm_x86_avx512_pmovzxbw(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovzxbw: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovzxbw {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero,mem[16],zero,mem[17],zero,mem[18],zero,mem[19],zero,mem[20],zero,mem[21],zero,mem[22],zero,mem[23],zero,mem[24],zero,mem[25],zero,mem[26],zero,mem[27],zero,mem[28],zero,mem[29],zero,mem[30],zero,mem[31],zero -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovzxbw: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovzxbw {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero,mem[16],zero,mem[17],zero,mem[18],zero,mem[19],zero,mem[20],zero,mem[21],zero,mem[22],zero,mem[23],zero,mem[24],zero,mem[25],zero,mem[26],zero,mem[27],zero,mem[28],zero,mem[29],zero,mem[30],zero,mem[31],zero +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovzxbw: ; X64: ## %bb.0: @@ -116,11 +116,11 @@ define <32 x i16> @test_llvm_x86_avx512_pmovzxbw(ptr %a) { } define <16 x i32> @test_llvm_x86_avx512_pmovzxbd(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovzxbd: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovzxbd {{.*#+}} zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovzxbd: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovzxbd {{.*#+}} zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovzxbd: ; X64: ## %bb.0: @@ -132,11 +132,11 @@ define <16 x i32> @test_llvm_x86_avx512_pmovzxbd(ptr %a) { } define <8 x i64> @test_llvm_x86_avx512_pmovzxbq(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovzxbq: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovzxbq {{.*#+}} zmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovzxbq: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovzxbq {{.*#+}} zmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovzxbq: ; X64: ## %bb.0: @@ -149,11 +149,11 @@ define <8 x i64> @test_llvm_x86_avx512_pmovzxbq(ptr %a) { } define <16 x i32> @test_llvm_x86_avx512_pmovzxwd(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovzxwd: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovzxwd {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovzxwd: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovzxwd {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovzxwd: ; X64: ## %bb.0: @@ -165,11 +165,11 @@ define <16 x i32> @test_llvm_x86_avx512_pmovzxwd(ptr %a) { } define <8 x i64> @test_llvm_x86_avx512_pmovzxwq(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovzxwq: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovzxwq {{.*#+}} zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovzxwq: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovzxwq {{.*#+}} zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovzxwq: ; X64: ## %bb.0: @@ -181,11 +181,11 @@ define <8 x i64> @test_llvm_x86_avx512_pmovzxwq(ptr %a) { } define <8 x i64> @test_llvm_x86_avx512_pmovzxdq(ptr %a) { -; X32-LABEL: test_llvm_x86_avx512_pmovzxdq: -; X32: ## %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: vpmovzxdq {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero -; X32-NEXT: retl +; X86-LABEL: test_llvm_x86_avx512_pmovzxdq: +; X86: ## %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: vpmovzxdq {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero +; X86-NEXT: retl ; ; X64-LABEL: test_llvm_x86_avx512_pmovzxdq: ; X64: ## %bb.0: -- GitLab From db9a16eaedd67f5b7c2dad68f1e3f2799d86d590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Thu, 4 Jan 2024 17:24:22 +0000 Subject: [PATCH 015/728] [mlir][nfc] Update comments in the Linalg vectoriser (#76797) --- .../Linalg/Transforms/Vectorization.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp index be813df8e782..5d99951ef09a 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp @@ -894,10 +894,16 @@ static bool isContiguousLoadIdx(LinalgOp &linalgOp, Value &val, return result; } -/// Check whether \p extractOp would be a gather or a contiguous load Op after -/// vectorising \p linalgOp. Note that it is always safe to use gather load -/// operations for contiguous loads (albeit slow), but not vice-versa. When in -/// doubt, bail out and assume that \p extractOp is a gather load. +/// Infer the memory access pattern for the input ExtractOp +/// +/// Based on the operation shapes and indices (usually based on the iteration +/// space of the parent `linalgOp` operation), decides whether the input +/// ExtractOp is a contiguous load (including a broadcast of a scalar) or a +/// gather load. +/// +/// Note that it is always safe to use gather load operations for contiguous +/// loads (albeit slow), but not vice-versa. When in doubt, bail out and assume +/// that `extractOp` is a gather load. static VectorMemoryAccessKind getTensorExtractMemoryAccessPattern(tensor::ExtractOp extractOp, LinalgOp &linalgOp) { @@ -916,8 +922,8 @@ getTensorExtractMemoryAccessPattern(tensor::ExtractOp extractOp, return VectorMemoryAccessKind::Gather; // 1. Assume that it's a gather load when reading _into_: - // * an n-D vector, like`tensor<1x2x4xi32` or`tensor<2x1x4xi32>`, or - // * a 1-D vector with the trailing dim equal 1, e.g. `tensor<1x4x1xi32`. + // * an n-D "vector", like `tensor<1x2x4xi32` or `tensor<2x1x4xi32>`, or + // * a 1-D "vector" with the trailing dim equal 1, e.g. `tensor<1x4x1xi32`. // TODO: Relax these conditions. // FIXME: This condition assumes non-dynamic sizes. if ((llvm::count_if(targetShape, -- GitLab From 79a2e2b9e8d0dffa602375c2386371666b412272 Mon Sep 17 00:00:00 2001 From: Nishant Mittal Date: Thu, 4 Jan 2024 22:54:54 +0530 Subject: [PATCH 016/728] [libc][math] Fix `is_quiet_nan` function in FPBits (#76931) --- libc/src/__support/FPUtil/FPBits.h | 8 ++++---- libc/src/__support/FPUtil/x86_64/LongDoubleBits.h | 4 ++++ libc/test/src/__support/FPUtil/fpbits_test.cpp | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h index 63eeba85f15e..93e32ba7cc94 100644 --- a/libc/src/__support/FPUtil/FPBits.h +++ b/libc/src/__support/FPUtil/FPBits.h @@ -162,9 +162,9 @@ protected: ? bit_at(SIG_LEN - 1) | bit_at(SIG_LEN - 2) // 0b1100... : bit_at(SIG_LEN - 1); // 0b1000... - // If a number x is a NAN, then it is a signalling NAN if: - // SIGNALING_NAN_MASK & bits(x) != 0 - LIBC_INLINE_VAR static constexpr StorageType SIGNALING_NAN_MASK = + // Mask to generate a default signaling NAN. Any NAN that is not + // a quiet NAN is considered a signaling NAN. + LIBC_INLINE_VAR static constexpr StorageType DEFAULT_SIGNALING_NAN = fp_type == FPType::X86_Binary80 ? bit_at(SIG_LEN - 1) | bit_at(SIG_LEN - 3) // 0b1010... : bit_at(SIG_LEN - 2); // 0b0100... @@ -356,7 +356,7 @@ public: } LIBC_INLINE constexpr bool is_quiet_nan() const { - return (bits & EXP_SIG_MASK) == (EXP_MASK | QUIET_NAN_MASK); + return (bits & EXP_SIG_MASK) >= (EXP_MASK | QUIET_NAN_MASK); } LIBC_INLINE constexpr bool is_inf_or_nan() const { diff --git a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h index 8abc0c87af0d..c18abcee77ea 100644 --- a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h +++ b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h @@ -114,6 +114,10 @@ public: (get_biased_exponent() != 0 && get_implicit_bit() == 0); } + LIBC_INLINE constexpr bool is_quiet_nan() const { + return (bits & EXP_SIG_MASK) >= (EXP_MASK | QUIET_NAN_MASK); + } + // Methods below this are used by tests. LIBC_INLINE static constexpr long double zero(bool sign = false) { diff --git a/libc/test/src/__support/FPUtil/fpbits_test.cpp b/libc/test/src/__support/FPUtil/fpbits_test.cpp index fa743855c486..e2dbe248ef21 100644 --- a/libc/test/src/__support/FPUtil/fpbits_test.cpp +++ b/libc/test/src/__support/FPUtil/fpbits_test.cpp @@ -69,6 +69,9 @@ TEST(LlvmLibcFPBitsTest, FloatType) { EXPECT_EQ(negnum.uintval(), static_cast(0xBF900000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), "0xBF900000 = (S: 1, E: 0x007F, M: 0x00100000)"); + + FloatBits quiet_nan = FloatBits(FloatBits::build_quiet_nan(1)); + EXPECT_EQ(quiet_nan.is_quiet_nan(), true); } TEST(LlvmLibcFPBitsTest, DoubleType) { @@ -129,6 +132,9 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { EXPECT_EQ(negnum.uintval(), static_cast(0xBFF2000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), "0xBFF2000000000000 = (S: 1, E: 0x03FF, M: 0x0002000000000000)"); + + DoubleBits quiet_nan = DoubleBits(DoubleBits::build_quiet_nan(1)); + EXPECT_EQ(quiet_nan.is_quiet_nan(), true); } #ifdef LIBC_TARGET_ARCH_IS_X86 @@ -210,6 +216,9 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LIBC_NAMESPACE::str(negnum).c_str(), "0x000000000000BFFF9000000000000000 = " "(S: 1, E: 0x3FFF, I: 1, M: 0x00000000000000001000000000000000)"); + + LongDoubleBits quiet_nan = LongDoubleBits(LongDoubleBits::build_quiet_nan(1)); + EXPECT_EQ(quiet_nan.is_quiet_nan(), true); } #else TEST(LlvmLibcFPBitsTest, LongDoubleType) { @@ -284,6 +293,9 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), "0xBFFF2000000000000000000000000000 = " "(S: 1, E: 0x3FFF, M: 0x00002000000000000000000000000000)"); + + LongDoubleBits quiet_nan = LongDoubleBits(LongDoubleBits::build_quiet_nan(1)); + EXPECT_EQ(quiet_nan.is_quiet_nan(), true); #endif } #endif @@ -357,5 +369,8 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), "0xBFFF2000000000000000000000000000 = " "(S: 1, E: 0x3FFF, M: 0x00002000000000000000000000000000)"); + + Float128Bits quiet_nan = Float128Bits(Float128Bits::build_quiet_nan(1)); + EXPECT_EQ(quiet_nan.is_quiet_nan(), true); } #endif // LIBC_COMPILER_HAS_FLOAT128 -- GitLab From 9f9dd6be0d21a156dcfee01ebbd571eca79b08bd Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Thu, 4 Jan 2024 09:28:45 -0800 Subject: [PATCH 017/728] Wrap local type declarations in anonymous namespace to fix modules build. --- lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp b/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp index 5326a73166e7..1688fb27430a 100644 --- a/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp +++ b/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp @@ -35,6 +35,7 @@ using namespace lldb_private; LLDB_PLUGIN_DEFINE(JITLoaderGDB) +namespace { // Debug Interface Structures enum jit_actions_t { JIT_NOACTION = 0, JIT_REGISTER_FN, JIT_UNREGISTER_FN }; @@ -52,7 +53,6 @@ template struct jit_descriptor { ptr_t first_entry; // pointer }; -namespace { enum EnableJITLoaderGDB { eEnableJITLoaderGDBDefault, eEnableJITLoaderGDBOn, -- GitLab From e78a1f491cbc0a57de7bf86058359dd0bd282540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E9=9B=A8=E5=9F=B9?= Date: Fri, 5 Jan 2024 01:32:10 +0800 Subject: [PATCH 018/728] [Clang] Fix the instantiation of return type requirements in lambda bodies (#76967) Currently, due to the incomplete implementation of p0588r1, the instantiation of lambda expressions leads to the instantiation of the body. And `EvaluateConstraints` is false during the instantiation of the body, which causes crashes during the instantiation of the return type requirement: ```cpp template concept doesnt_matter = true; template concept test = []{ return requires(T t) { { t } -> doesnt_matter; // crash }; }(); static_assert(test); ``` Although a complete implementation of p0588r1 can solve these crashes, it will take some time. Therefore, this pull request aims to fix these crashes first. Fixes https://github.com/llvm/llvm-project/issues/63808 Fixes https://github.com/llvm/llvm-project/issues/64607 Fixes https://github.com/llvm/llvm-project/issues/64086 --- clang/docs/ReleaseNotes.rst | 5 ++++ clang/lib/Sema/SemaTemplateInstantiate.cpp | 17 +++++++++++ clang/test/SemaTemplate/concepts-lambda.cpp | 33 +++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c7bf162426a6..e5de042cebd4 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -844,6 +844,11 @@ Bug Fixes to C++ Support - Fix crash when parsing nested requirement. Fixes: (`#73112 `_) +- Fixed a crash caused by using return type requirement in a lambda. Fixes: + (`#63808 `_) + (`#64607 `_) + (`#64086 `_) + Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ - Fixed an import failure of recursive friend class template. diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp index 37e5b9cad08b..e1cbdcd72eac 100644 --- a/clang/lib/Sema/SemaTemplateInstantiate.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp @@ -1479,6 +1479,23 @@ namespace { return Result; } + StmtResult TransformLambdaBody(LambdaExpr *E, Stmt *Body) { + // Currently, we instantiate the body when instantiating the lambda + // expression. However, `EvaluateConstraints` is disabled during the + // instantiation of the lambda expression, causing the instantiation + // failure of the return type requirement in the body. If p0588r1 is fully + // implemented, the body will be lazily instantiated, and this problem + // will not occur. Here, `EvaluateConstraints` is temporarily set to + // `true` to temporarily fix this issue. + // FIXME: This temporary fix can be removed after fully implementing + // p0588r1. + bool Prev = EvaluateConstraints; + EvaluateConstraints = true; + StmtResult Stmt = inherited::TransformLambdaBody(E, Body); + EvaluateConstraints = Prev; + return Stmt; + } + ExprResult TransformRequiresExpr(RequiresExpr *E) { LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true); ExprResult TransReq = inherited::TransformRequiresExpr(E); diff --git a/clang/test/SemaTemplate/concepts-lambda.cpp b/clang/test/SemaTemplate/concepts-lambda.cpp index 8a184cbf4e9b..7e431529427d 100644 --- a/clang/test/SemaTemplate/concepts-lambda.cpp +++ b/clang/test/SemaTemplate/concepts-lambda.cpp @@ -116,3 +116,36 @@ static_assert(E); // expected-note@-11{{because 'Q.template operator()()' would be invalid: no matching member function for call to 'operator()'}} } } + +namespace ReturnTypeRequirementInLambda { +template +concept C1 = true; + +template +concept test = [] { + return requires(T t) { + { t } -> C1; + }; +}(); + +static_assert(test); + +template +concept C2 = true; +struct S1 { + int f1() { return 1; } +}; + +void foo() { + auto make_caller = [] { + return [](S1 *ps) { + if constexpr (requires { + { (ps->*member)() } -> C2; + }) + ; + }; + }; + + auto caller = make_caller.operator()<&S1::f1>(); +} +} // namespace ReturnTypeRequirementInLambda -- GitLab From 4004f655ceb9623608ba0471aa7037c142956e31 Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Thu, 4 Jan 2024 12:27:03 -0500 Subject: [PATCH 019/728] [LLDB][NativePDB] Fix use-after-free error detected by asan. --- .../Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp index 8375010ae3de..9234768323e7 100644 --- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp +++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp @@ -2265,7 +2265,8 @@ void SymbolFileNativePDB::BuildParentMap() { } for (TypeIndex fwd : fwd_keys) { TypeIndex full = forward_to_full[fwd]; - m_parent_types[full] = m_parent_types[fwd]; + TypeIndex parent_idx = m_parent_types[fwd]; + m_parent_types[full] = parent_idx; } for (TypeIndex full : full_keys) { TypeIndex fwd = full_to_forward[full]; -- GitLab From a960703466e937d99ab7a7a29f7448e1bc926e35 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 4 Jan 2024 09:48:40 -0800 Subject: [PATCH 020/728] [RISCV] Remove incomplete PRE_DEC/POST_DEC code for XTHeadMemIdx. (#76922) As far as I can tell if getIndexedAddressParts received an ISD::SUB, the constant would be negated. So `IsInc` should be set to true since the SUB was effectively converted to ADD. This means we should never use PRE_DEC/POST_DEC. No tests are affected because DAGCombine aggressively turns SUB with constant into ADD so no lit test has a SUB reach getIndexedAddressParts. --- llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 10 ++++------ llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 15 +++++---------- llvm/lib/Target/RISCV/RISCVISelLowering.h | 3 +-- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index bfa3bf3cc74e..7257c2e8fe1f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -763,14 +763,12 @@ bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) { return false; EVT LoadVT = Ld->getMemoryVT(); - bool IsPre = (AM == ISD::PRE_INC || AM == ISD::PRE_DEC); - bool IsPost = (AM == ISD::POST_INC || AM == ISD::POST_DEC); + assert(AM == ISD::PRE_INC || + AM == ISD::POST_INC && "Unexpected addressing mode"); + bool IsPre = AM == ISD::PRE_INC; + bool IsPost = AM == ISD::POST_INC; int64_t Offset = C->getSExtValue(); - // Convert decrements to increments by a negative quantity. - if (AM == ISD::PRE_DEC || AM == ISD::POST_DEC) - Offset = -Offset; - // The constants that can be encoded in the THeadMemIdx instructions // are of the form (sign_extend(imm5) << imm2). int64_t Shift; diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index ae0e6ef10132..bc4b2b022c0a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1350,8 +1350,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, } if (Subtarget.hasVendorXTHeadMemIdx()) { - for (unsigned im = (unsigned)ISD::PRE_INC; im != (unsigned)ISD::POST_DEC; - ++im) { + for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) { setIndexedLoadAction(im, MVT::i8, Legal); setIndexedStoreAction(im, MVT::i8, Legal); setIndexedLoadAction(im, MVT::i16, Legal); @@ -19269,7 +19268,6 @@ bool RISCVTargetLowering::isVScaleKnownToBeAPowerOfTwo() const { bool RISCVTargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, - bool &IsInc, SelectionDAG &DAG) const { // Target does not support indexed loads. if (!Subtarget.hasVendorXTHeadMemIdx()) @@ -19296,7 +19294,6 @@ bool RISCVTargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base, if (!isLegalIndexedOffset) return false; - IsInc = (Op->getOpcode() == ISD::ADD); Offset = Op->getOperand(1); return true; } @@ -19319,11 +19316,10 @@ bool RISCVTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, } else return false; - bool IsInc; - if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, IsInc, DAG)) + if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, DAG)) return false; - AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC; + AM = ISD::PRE_INC; return true; } @@ -19343,15 +19339,14 @@ bool RISCVTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op, } else return false; - bool IsInc; - if (!getIndexedAddressParts(Op, Base, Offset, AM, IsInc, DAG)) + if (!getIndexedAddressParts(Op, Base, Offset, AM, DAG)) return false; // Post-indexing updates the base, so it's not a valid transform // if that's not the same as the load's pointer. if (Ptr != Base) return false; - AM = IsInc ? ISD::POST_INC : ISD::POST_DEC; + AM = ISD::POST_INC; return true; } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 226b51ddda6e..18f580575581 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -774,8 +774,7 @@ public: bool isVScaleKnownToBeAPowerOfTwo() const override; bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, - ISD::MemIndexedMode &AM, bool &IsInc, - SelectionDAG &DAG) const; + ISD::MemIndexedMode &AM, SelectionDAG &DAG) const; bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override; -- GitLab From 4f59a38821a2175408a3189327223b85ddba636f Mon Sep 17 00:00:00 2001 From: Pete Steinfeld <47540744+psteinfeld@users.noreply.github.com> Date: Thu, 4 Jan 2024 10:19:50 -0800 Subject: [PATCH 021/728] Revert #76194 (#76987) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [Flang] Revert "Allow Intrinsic simpification with min/maxloc dim and…scalar result (#76194)" This reverts commit 9b7cf5bfb08b6e506216ef354dfd61adb15acbff. See merge request #76194. This change was causing several failures in our internal tests. I'm reverting now and will work on creating a test that David Green can use to reproduce the problem. --- .../Transforms/SimplifyIntrinsics.cpp | 15 ++-- flang/test/Transforms/simplifyintrinsics.fir | 68 ++----------------- 2 files changed, 13 insertions(+), 70 deletions(-) diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp index f5ddcbbaecd2..c89ee6d5e203 100644 --- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp +++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp @@ -1162,14 +1162,11 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( mlir::Operation::operand_range args = call.getArgs(); - mlir::SymbolRefAttr callee = call.getCalleeAttr(); - mlir::StringRef funcNameBase = callee.getLeafReference().getValue(); - bool isDim = funcNameBase.ends_with("Dim"); - mlir::Value back = args[isDim ? 7 : 6]; + mlir::Value back = args[6]; if (isTrueOrNotConstant(back)) return; - mlir::Value mask = args[isDim ? 6 : 5]; + mlir::Value mask = args[5]; mlir::Value maskDef = findMaskDef(mask); // maskDef is set to NULL when the defining op is not one we accept. @@ -1178,8 +1175,10 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( if (maskDef == NULL) return; + mlir::SymbolRefAttr callee = call.getCalleeAttr(); + mlir::StringRef funcNameBase = callee.getLeafReference().getValue(); unsigned rank = getDimCount(args[1]); - if ((isDim && rank != 1) || !(rank > 0)) + if (funcNameBase.ends_with("Dim") || !(rank > 0)) return; fir::FirOpBuilder builder{getSimplificationBuilder(call, kindMap)}; @@ -1220,8 +1219,6 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( llvm::raw_string_ostream nameOS(funcName); outType.print(nameOS); - if (isDim) - nameOS << '_' << inputType; nameOS << '_' << fmfString; auto typeGenerator = [rank](fir::FirOpBuilder &builder) { @@ -1237,7 +1234,7 @@ void SimplifyIntrinsicsPass::simplifyMinMaxlocReduction( mlir::func::FuncOp newFunc = getOrCreateFunction(builder, funcName, typeGenerator, bodyGenerator); builder.create(loc, newFunc, - mlir::ValueRange{args[0], args[1], mask}); + mlir::ValueRange{args[0], args[1], args[5]}); call->dropAllReferences(); call->erase(); } diff --git a/flang/test/Transforms/simplifyintrinsics.fir b/flang/test/Transforms/simplifyintrinsics.fir index 61cddd4f48df..0bd6ac7c436f 100644 --- a/flang/test/Transforms/simplifyintrinsics.fir +++ b/flang/test/Transforms/simplifyintrinsics.fir @@ -2115,13 +2115,13 @@ func.func @_QPtestminloc_doesntwork1d_back(%arg0: !fir.ref> { // CHECK-NOT: fir.call @_FortranAMinlocInteger4x1_i32_contract_simplified({{.*}}) fastmath : (!fir.ref>, !fir.box, !fir.box) -> () // ----- -// Check Minloc is simplified when DIM arg is set so long as the result is scalar +// Check Minloc is not simplified when DIM arg is set -func.func @_QPtestminloc_1d_dim(%arg0: !fir.ref> {fir.bindc_name = "a"}) -> !fir.array<1xi32> { +func.func @_QPtestminloc_doesntwork1d_dim(%arg0: !fir.ref> {fir.bindc_name = "a"}) -> !fir.array<1xi32> { %0 = fir.alloca !fir.box> %c10 = arith.constant 10 : index %c1 = arith.constant 1 : index - %1 = fir.alloca !fir.array<1xi32> {bindc_name = "testminloc_1d_dim", uniq_name = "_QFtestminloc_1d_dimEtestminloc_1d_dim"} + %1 = fir.alloca !fir.array<1xi32> {bindc_name = "testminloc_doesntwork1d_dim", uniq_name = "_QFtestminloc_doesntwork1d_dimEtestminloc_doesntwork1d_dim"} %2 = fir.shape %c1 : (index) -> !fir.shape<1> %3 = fir.array_load %1(%2) : (!fir.ref>, !fir.shape<1>) -> !fir.array<1xi32> %4 = fir.shape %c10 : (index) -> !fir.shape<1> @@ -2156,65 +2156,11 @@ func.func @_QPtestminloc_1d_dim(%arg0: !fir.ref> {fir.bindc_n %21 = fir.load %1 : !fir.ref> return %21 : !fir.array<1xi32> } -// CHECK-LABEL: func.func @_QPtestminloc_1d_dim( +// CHECK-LABEL: func.func @_QPtestminloc_doesntwork1d_dim( // CHECK-SAME: %[[ARR:.*]]: !fir.ref> {fir.bindc_name = "a"}) -> !fir.array<1xi32> { -// CHECK: fir.call @_FortranAMinlocDimx1_i32_i32_contract_simplified({{.*}}) fastmath : (!fir.ref>, !fir.box, !fir.box) -> () - -// CHECK-LABEL: func.func private @_FortranAMinlocDimx1_i32_i32_contract_simplified(%arg0: !fir.ref>, %arg1: !fir.box, %arg2: !fir.box) attributes {llvm.linkage = #llvm.linkage} { -// CHECK-NEXT: %[[V0:.*]] = fir.alloca i32 -// CHECK-NEXT: %c0_i32 = arith.constant 0 : i32 -// CHECK-NEXT: %c1 = arith.constant 1 : index -// CHECK-NEXT: %[[V1:.*]] = fir.allocmem !fir.array<1xi32> -// CHECK-NEXT: %[[V2:.*]] = fir.shape %c1 : (index) -> !fir.shape<1> -// CHECK-NEXT: %[[V3:.*]] = fir.embox %[[V1]](%[[V2]]) : (!fir.heap>, !fir.shape<1>) -> !fir.box>> -// CHECK-NEXT: %c0 = arith.constant 0 : index -// CHECK-NEXT: %[[V4:.*]] = fir.coordinate_of %[[V3]], %c0 : (!fir.box>>, index) -> !fir.ref -// CHECK-NEXT: fir.store %c0_i32 to %[[V4]] : !fir.ref -// CHECK-NEXT: %c0_0 = arith.constant 0 : index -// CHECK-NEXT: %[[V5:.*]] = fir.convert %arg1 : (!fir.box) -> !fir.box> -// CHECK-NEXT: %c1_i32 = arith.constant 1 : i32 -// CHECK-NEXT: %c0_i32_1 = arith.constant 0 : i32 -// CHECK-NEXT: fir.store %c0_i32_1 to %[[V0]] : !fir.ref -// CHECK-NEXT: %c2147483647_i32 = arith.constant 2147483647 : i32 -// CHECK-NEXT: %c1_2 = arith.constant 1 : index -// CHECK-NEXT: %c0_3 = arith.constant 0 : index -// CHECK-NEXT: %[[V6:.*]]:3 = fir.box_dims %[[V5]], %c0_3 : (!fir.box>, index) -> (index, index, index) -// CHECK-NEXT: %[[V7:.*]] = arith.subi %[[V6]]#1, %c1_2 : index -// CHECK-NEXT: %[[V8:.*]] = fir.do_loop %arg3 = %c0_0 to %[[V7]] step %c1_2 iter_args(%arg4 = %c2147483647_i32) -> (i32) { -// CHECK-NEXT: fir.store %c1_i32 to %[[V0]] : !fir.ref -// CHECK-NEXT: %[[V12:.*]] = fir.coordinate_of %[[V5]], %arg3 : (!fir.box>, index) -> !fir.ref -// CHECK-NEXT: %[[V13:.*]] = fir.load %[[V12]] : !fir.ref -// CHECK-NEXT: %[[V14:.*]] = arith.cmpi slt, %[[V13]], %arg4 : i32 -// CHECK-NEXT: %[[V15:.*]] = fir.if %[[V14]] -> (i32) { -// CHECK-NEXT: %c1_i32_4 = arith.constant 1 : i32 -// CHECK-NEXT: %c0_5 = arith.constant 0 : index -// CHECK-NEXT: %[[V16:.*]] = fir.coordinate_of %[[V3]], %c0_5 : (!fir.box>>, index) -> !fir.ref -// CHECK-NEXT: %[[V17:.*]] = fir.convert %arg3 : (index) -> i32 -// CHECK-NEXT: %[[V18:.*]] = arith.addi %[[V17]], %c1_i32_4 : i32 -// CHECK-NEXT: fir.store %[[V18]] to %[[V16]] : !fir.ref -// CHECK-NEXT: fir.result %[[V13]] : i32 -// CHECK-NEXT: } else { -// CHECK-NEXT: fir.result %arg4 : i32 -// CHECK-NEXT: } -// CHECK-NEXT: fir.result %[[V15]] : i32 -// CHECK-NEXT: } -// CHECK-NEXT: %[[V9:.*]] = fir.load %[[V0]] : !fir.ref -// CHECK-NEXT: %[[V10:.*]] = arith.cmpi eq, %[[V9]], %c1_i32 : i32 -// CHECK-NEXT: fir.if %[[V10]] { -// CHECK-NEXT: %c2147483647_i32_4 = arith.constant 2147483647 : i32 -// CHECK-NEXT: %[[V12]] = arith.cmpi eq, %c2147483647_i32_4, %[[V8]] : i32 -// CHECK-NEXT: fir.if %[[V12]] { -// CHECK-NEXT: %c0_5 = arith.constant 0 : index -// CHECK-NEXT: %[[V13]] = fir.coordinate_of %[[V3]], %c0_5 : (!fir.box>>, index) -> !fir.ref -// CHECK-NEXT: fir.store %c1_i32 to %[[V13]] : !fir.ref -// CHECK-NEXT: } -// CHECK-NEXT: } -// CHECK-NEXT: %[[V11:.*]] = fir.convert %arg0 : (!fir.ref>) -> !fir.ref>>> -// CHECK-NEXT: fir.store %[[V3]] to %[[V11]] : !fir.ref>>> -// CHECK-NEXT: return -// CHECK-NEXT: } - - +// CHECK-NOT: fir.call @_FortranAMinlocDimx1_i32_contract_simplified({{.*}}) fastmath : (!fir.ref>, !fir.box, !fir.box) -> () +// CHECK: fir.call @_FortranAMinlocDim({{.*}}) fastmath : (!fir.ref>, !fir.box, i32, i32, !fir.ref, i32, !fir.box, i1) -> none +// CHECK-NOT: fir.call @_FortranAMinlocDimx1_i32_contract_simplified({{.*}}) fastmath : (!fir.ref>, !fir.box, !fir.box) -> () // ----- // Check Minloc is not simplified when dimension of inputArr is unknown -- GitLab From c041fa1093c3ad7be040fb362a10ca3900c698a4 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Thu, 4 Jan 2024 10:25:27 -0800 Subject: [PATCH 022/728] XFAIL test with dsymutil --- .../functionalities/inline-sourcefile/TestInlineSourceFiles.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py b/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py index 20ed0ce00661..ce7ac6fc503e 100644 --- a/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py +++ b/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py @@ -8,6 +8,8 @@ from lldbsuite.test import lldbutil class InlineSourceFilesTestCase(TestBase): @skipIf(compiler="gcc") @skipIf(compiler="clang", compiler_version=["<", "18.0"]) + # dsymutil doesn't yet copy the sources + @expectedFailureDarwin(debug_info=["dsym"]) def test(self): """Test DWARF inline source files.""" self.build() -- GitLab From 85939e5e248213dfdf66fc8305ed502fc2f3f1f0 Mon Sep 17 00:00:00 2001 From: Valentin Clement Date: Thu, 4 Jan 2024 10:23:38 -0800 Subject: [PATCH 023/728] [mlir][openacc][NFC] Rename custom parser from WaitOperands to DeviceTypeOperandsWithSegment --- mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td | 9 +++++---- mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp | 11 +++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index 234c1076e14e..4312bd4de1bd 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -976,7 +976,7 @@ def OpenACC_ParallelOp : OpenACC_Op<"parallel", `)` | `vector_length` `(` custom($vectorLength, type($vectorLength), $vectorLengthDeviceType) `)` - | `wait` `(` custom($waitOperands, + | `wait` `(` custom($waitOperands, type($waitOperands), $waitOperandsDeviceType, $waitOperandsSegments) `)` | `self` `(` $selfCond `)` | `if` `(` $ifCond `)` @@ -1075,7 +1075,7 @@ def OpenACC_SerialOp : OpenACC_Op<"serial", | `private` `(` custom( $gangPrivateOperands, type($gangPrivateOperands), $privatizations) `)` - | `wait` `(` custom($waitOperands, + | `wait` `(` custom($waitOperands, type($waitOperands), $waitOperandsDeviceType, $waitOperandsSegments) `)` | `self` `(` $selfCond `)` | `if` `(` $ifCond `)` @@ -1196,8 +1196,9 @@ def OpenACC_KernelsOp : OpenACC_Op<"kernels", type($numWorkers), $numWorkersDeviceType) `)` | `vector_length` `(` custom($vectorLength, type($vectorLength), $vectorLengthDeviceType) `)` - | `wait` `(` custom($waitOperands, - type($waitOperands), $waitOperandsDeviceType, $waitOperandsSegments) `)` + | `wait` `(` custom($waitOperands, + type($waitOperands), $waitOperandsDeviceType, + $waitOperandsSegments) `)` | `self` `(` $selfCond `)` | `if` `(` $ifCond `)` ) diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp index f484eda3268d..e299b67b10a9 100644 --- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp +++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp @@ -874,7 +874,7 @@ static void printNumGangs(mlir::OpAsmPrinter &p, mlir::Operation *op, } } -static ParseResult parseWaitOperands( +static ParseResult parseDeviceTypeOperandsWithSegment( mlir::OpAsmParser &parser, llvm::SmallVectorImpl &operands, llvm::SmallVectorImpl &types, mlir::ArrayAttr &deviceTypes, @@ -918,11 +918,10 @@ static ParseResult parseWaitOperands( return success(); } -static void printWaitOperands(mlir::OpAsmPrinter &p, mlir::Operation *op, - mlir::OperandRange operands, - mlir::TypeRange types, - std::optional deviceTypes, - std::optional segments) { +static void printDeviceTypeOperandsWithSegment( + mlir::OpAsmPrinter &p, mlir::Operation *op, mlir::OperandRange operands, + mlir::TypeRange types, std::optional deviceTypes, + std::optional segments) { unsigned opIdx = 0; for (unsigned i = 0; i < deviceTypes->size(); ++i) { if (i != 0) -- GitLab From 8b2bdfbca7c1db272e4e703445f5626b4bc4b9d3 Mon Sep 17 00:00:00 2001 From: Alan Phipps Date: Mon, 1 Jan 2024 11:40:55 -0600 Subject: [PATCH 024/728] [Coverage][clang] Enable MC/DC Support in LLVM Source-based Code Coverage (3/3) Part 3 of 3. This includes the MC/DC clang front-end components. Differential Revision: https://reviews.llvm.org/D138849 --- clang/include/clang/Basic/CodeGenOptions.def | 1 + clang/include/clang/Driver/Options.td | 6 + clang/lib/CodeGen/CGClass.cpp | 6 +- clang/lib/CodeGen/CGExprScalar.cpp | 57 +++ clang/lib/CodeGen/CGStmt.cpp | 14 +- clang/lib/CodeGen/CodeGenFunction.cpp | 98 +++-- clang/lib/CodeGen/CodeGenFunction.h | 58 ++- clang/lib/CodeGen/CodeGenPGO.cpp | 263 +++++++++++- clang/lib/CodeGen/CodeGenPGO.h | 14 +- clang/lib/CodeGen/CoverageMappingGen.cpp | 405 ++++++++++++++++-- clang/lib/CodeGen/CoverageMappingGen.h | 12 +- clang/lib/Driver/ToolChains/Clang.cpp | 11 + .../CoverageMapping/branch-constfolded.cpp | 42 +- .../CoverageMapping/branch-mincounters.cpp | 1 + .../test/CoverageMapping/branch-templates.cpp | 1 + clang/test/CoverageMapping/if.cpp | 1 + clang/test/CoverageMapping/logical.cpp | 10 +- clang/test/CoverageMapping/mcdc-class.cpp | 31 ++ .../CoverageMapping/mcdc-error-conditions.cpp | 7 + .../test/CoverageMapping/mcdc-error-nests.cpp | 10 + .../mcdc-logical-scalar-ids.cpp | 110 +++++ .../mcdc-logical-stmt-ids-all.cpp | 131 ++++++ .../CoverageMapping/mcdc-logical-stmt-ids.cpp | 111 +++++ .../Profile/c-linkage-available_externally.c | 2 + clang/test/Profile/c-mcdc-class.cpp | 104 +++++ clang/test/Profile/c-mcdc-nested-ternary.c | 68 +++ clang/test/Profile/c-mcdc-not.c | 88 ++++ clang/test/Profile/c-mcdc.c | 102 +++++ .../ContinuousSyncMode/image-with-mcdc.c | 26 ++ 29 files changed, 1700 insertions(+), 90 deletions(-) create mode 100644 clang/test/CoverageMapping/mcdc-class.cpp create mode 100644 clang/test/CoverageMapping/mcdc-error-conditions.cpp create mode 100644 clang/test/CoverageMapping/mcdc-error-nests.cpp create mode 100644 clang/test/CoverageMapping/mcdc-logical-scalar-ids.cpp create mode 100644 clang/test/CoverageMapping/mcdc-logical-stmt-ids-all.cpp create mode 100644 clang/test/CoverageMapping/mcdc-logical-stmt-ids.cpp create mode 100644 clang/test/Profile/c-mcdc-class.cpp create mode 100644 clang/test/Profile/c-mcdc-nested-ternary.c create mode 100644 clang/test/Profile/c-mcdc-not.c create mode 100644 clang/test/Profile/c-mcdc.c create mode 100644 compiler-rt/test/profile/ContinuousSyncMode/image-with-mcdc.c diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def index 0acb5ae134ea..2c4fb6745bc1 100644 --- a/clang/include/clang/Basic/CodeGenOptions.def +++ b/clang/include/clang/Basic/CodeGenOptions.def @@ -209,6 +209,7 @@ CODEGENOPT(CoverageMapping , 1, 0) ///< Generate coverage mapping regions to ///< enable code coverage analysis. CODEGENOPT(DumpCoverageMapping , 1, 0) ///< Dump the generated coverage mapping ///< regions. +CODEGENOPT(MCDCCoverage , 1, 0) ///< Enable MC/DC code coverage criteria. /// If -fpcc-struct-return or -freg-struct-return is specified. ENUM_CODEGENOPT(StructReturnConvention, StructReturnConventionKind, 2, SRCK_Default) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 2b93ddf03349..6aff37f13368 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1695,6 +1695,12 @@ defm coverage_mapping : BoolFOption<"coverage-mapping", "Generate coverage mapping to enable code coverage analysis">, NegFlag, BothFlags< [], [ClangOption, CLOption]>>; +defm mcdc_coverage : BoolFOption<"coverage-mcdc", + CodeGenOpts<"MCDCCoverage">, DefaultFalse, + PosFlag, + NegFlag, + BothFlags<[], [ClangOption, CLOption]>>; def fprofile_generate : Flag<["-"], "fprofile-generate">, Group, Visibility<[ClangOption, CLOption]>, HelpText<"Generate instrumented code to collect execution counts into default.profraw (overridden by LLVM_PROFILE_FILE env var)">; diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index d18f186ce5b4..34319381901a 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -856,6 +856,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) { EnterCXXTryStmt(*cast(Body), true); incrementProfileCounter(Body); + maybeCreateMCDCCondBitmap(); RunCleanupsScope RunCleanups(*this); @@ -1444,8 +1445,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { } Stmt *Body = Dtor->getBody(); - if (Body) + if (Body) { incrementProfileCounter(Body); + maybeCreateMCDCCondBitmap(); + } // The call to operator delete in a deleting destructor happens // outside of the function-try-block, which means it's always @@ -1548,6 +1551,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) LexicalScope Scope(*this, RootCS->getSourceRange()); incrementProfileCounter(RootCS); + maybeCreateMCDCCondBitmap(); AssignmentMemcpyizer AM(*this, AssignOp, Args); for (auto *I : RootCS->body()) AM.emitAssignment(I); diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp index d2c4c7ee50bc..9ec185153d12 100644 --- a/clang/lib/CodeGen/CGExprScalar.cpp +++ b/clang/lib/CodeGen/CGExprScalar.cpp @@ -4564,6 +4564,12 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { if (LHSCondVal) { // If we have 1 && X, just emit X. CGF.incrementProfileCounter(E); + // If the top of the logical operator nest, reset the MCDC temp to 0. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeResetMCDCCondBitmap(E); + + CGF.MCDCLogOpStack.push_back(E); + Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); // If we're generating for profiling or coverage, generate a branch to a @@ -4572,6 +4578,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { // "FalseBlock" after the increment is done. if (InstrumentRegions && CodeGenFunction::isInstrumentedCondition(E->getRHS())) { + CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond); llvm::BasicBlock *FBlock = CGF.createBasicBlock("land.end"); llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("land.rhscnt"); Builder.CreateCondBr(RHSCond, RHSBlockCnt, FBlock); @@ -4581,6 +4588,11 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { CGF.EmitBlock(FBlock); } + CGF.MCDCLogOpStack.pop_back(); + // If the top of the logical operator nest, update the MCDC bitmap. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeUpdateMCDCTestVectorBitmap(E); + // ZExt result to int or bool. return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "land.ext"); } @@ -4590,6 +4602,12 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { return llvm::Constant::getNullValue(ResTy); } + // If the top of the logical operator nest, reset the MCDC temp to 0. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeResetMCDCCondBitmap(E); + + CGF.MCDCLogOpStack.push_back(E); + llvm::BasicBlock *ContBlock = CGF.createBasicBlock("land.end"); llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("land.rhs"); @@ -4622,6 +4640,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { // condition coverage. if (InstrumentRegions && CodeGenFunction::isInstrumentedCondition(E->getRHS())) { + CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond); llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("land.rhscnt"); Builder.CreateCondBr(RHSCond, RHSBlockCnt, ContBlock); CGF.EmitBlock(RHSBlockCnt); @@ -4639,6 +4658,11 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { // Insert an entry into the phi node for the edge with the value of RHSCond. PN->addIncoming(RHSCond, RHSBlock); + CGF.MCDCLogOpStack.pop_back(); + // If the top of the logical operator nest, update the MCDC bitmap. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeUpdateMCDCTestVectorBitmap(E); + // Artificial location to preserve the scope information { auto NL = ApplyDebugLocation::CreateArtificial(CGF); @@ -4680,6 +4704,12 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { if (!LHSCondVal) { // If we have 0 || X, just emit X. CGF.incrementProfileCounter(E); + // If the top of the logical operator nest, reset the MCDC temp to 0. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeResetMCDCCondBitmap(E); + + CGF.MCDCLogOpStack.push_back(E); + Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); // If we're generating for profiling or coverage, generate a branch to a @@ -4688,6 +4718,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { // "FalseBlock" after the increment is done. if (InstrumentRegions && CodeGenFunction::isInstrumentedCondition(E->getRHS())) { + CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond); llvm::BasicBlock *FBlock = CGF.createBasicBlock("lor.end"); llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("lor.rhscnt"); Builder.CreateCondBr(RHSCond, FBlock, RHSBlockCnt); @@ -4697,6 +4728,11 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { CGF.EmitBlock(FBlock); } + CGF.MCDCLogOpStack.pop_back(); + // If the top of the logical operator nest, update the MCDC bitmap. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeUpdateMCDCTestVectorBitmap(E); + // ZExt result to int or bool. return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "lor.ext"); } @@ -4706,6 +4742,12 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { return llvm::ConstantInt::get(ResTy, 1); } + // If the top of the logical operator nest, reset the MCDC temp to 0. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeResetMCDCCondBitmap(E); + + CGF.MCDCLogOpStack.push_back(E); + llvm::BasicBlock *ContBlock = CGF.createBasicBlock("lor.end"); llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("lor.rhs"); @@ -4742,6 +4784,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { // condition coverage. if (InstrumentRegions && CodeGenFunction::isInstrumentedCondition(E->getRHS())) { + CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond); llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("lor.rhscnt"); Builder.CreateCondBr(RHSCond, ContBlock, RHSBlockCnt); CGF.EmitBlock(RHSBlockCnt); @@ -4755,6 +4798,11 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { CGF.EmitBlock(ContBlock); PN->addIncoming(RHSCond, RHSBlock); + CGF.MCDCLogOpStack.pop_back(); + // If the top of the logical operator nest, update the MCDC bitmap. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeUpdateMCDCTestVectorBitmap(E); + // ZExt result to int. return Builder.CreateZExtOrBitCast(PN, ResTy, "lor.ext"); } @@ -4899,6 +4947,10 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { return Builder.CreateSelect(CondV, LHS, RHS, "cond"); } + // If the top of the logical operator nest, reset the MCDC temp to 0. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeResetMCDCCondBitmap(condExpr); + llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true"); llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false"); llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end"); @@ -4934,6 +4986,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { llvm::PHINode *PN = Builder.CreatePHI(LHS->getType(), 2, "cond"); PN->addIncoming(LHS, LHSBlock); PN->addIncoming(RHS, RHSBlock); + + // If the top of the logical operator nest, update the MCDC bitmap. + if (CGF.MCDCLogOpStack.empty()) + CGF.maybeUpdateMCDCTestVectorBitmap(condExpr); + return PN; } diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index 0f79a2e861d2..b89017de0bcf 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -837,7 +837,19 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { if (!ThenCount && !getCurrentProfileCount() && CGM.getCodeGenOpts().OptimizationLevel) LH = Stmt::getLikelihood(S.getThen(), S.getElse()); - EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH); + + // When measuring MC/DC, always fully evaluate the condition up front using + // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to + // executing the body of the if.then or if.else. This is useful for when + // there is a 'return' within the body, but this is particularly beneficial + // when one if-stmt is nested within another if-stmt so that all of the MC/DC + // updates are kept linear and consistent. + if (!CGM.getCodeGenOpts().MCDCCoverage) + EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH); + else { + llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); + Builder.CreateCondBr(BoolCondVal, ThenBlock, ElseBlock); + } // Emit the 'then' code. EmitBlock(ThenBlock); diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index 2199d7b58fb9..2673e4a5cee7 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -1256,6 +1256,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, void CodeGenFunction::EmitFunctionBody(const Stmt *Body) { incrementProfileCounter(Body); + maybeCreateMCDCCondBitmap(); if (const CompoundStmt *S = dyn_cast(Body)) EmitCompoundStmtWithoutScope(*S); else @@ -1601,6 +1602,13 @@ bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) { bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond, bool &ResultBool, bool AllowLabels) { + // If MC/DC is enabled, disable folding so that we can instrument all + // conditions to yield complete test vectors. We still keep track of + // folded conditions during region mapping and visualization. + if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() && + CGM.getCodeGenOpts().MCDCCoverage) + return false; + llvm::APSInt ResultInt; if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels)) return false; @@ -1629,16 +1637,20 @@ bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond, return true; } +/// Strip parentheses and simplistic logical-NOT operators. +const Expr *CodeGenFunction::stripCond(const Expr *C) { + while (const UnaryOperator *Op = dyn_cast(C->IgnoreParens())) { + if (Op->getOpcode() != UO_LNot) + break; + C = Op->getSubExpr(); + } + return C->IgnoreParens(); +} + /// Determine whether the given condition is an instrumentable condition /// (i.e. no "&&" or "||"). bool CodeGenFunction::isInstrumentedCondition(const Expr *C) { - // Bypass simplistic logical-NOT operator before determining whether the - // condition contains any other logical operator. - if (const UnaryOperator *UnOp = dyn_cast(C->IgnoreParens())) - if (UnOp->getOpcode() == UO_LNot) - C = UnOp->getSubExpr(); - - const BinaryOperator *BOp = dyn_cast(C->IgnoreParens()); + const BinaryOperator *BOp = dyn_cast(stripCond(C)); return (!BOp || !BOp->isLogicalOp()); } @@ -1717,17 +1729,19 @@ void CodeGenFunction::EmitBranchToCounterBlock( /// statement) to the specified blocks. Based on the condition, this might try /// to simplify the codegen of the conditional based on the branch. /// \param LH The value of the likelihood attribute on the True branch. -void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, - llvm::BasicBlock *TrueBlock, - llvm::BasicBlock *FalseBlock, - uint64_t TrueCount, - Stmt::Likelihood LH) { +/// \param ConditionalOp Used by MC/DC code coverage to track the result of the +/// ConditionalOperator (ternary) through a recursive call for the operator's +/// LHS and RHS nodes. +void CodeGenFunction::EmitBranchOnBoolExpr( + const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, + uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp) { Cond = Cond->IgnoreParens(); if (const BinaryOperator *CondBOp = dyn_cast(Cond)) { - // Handle X && Y in a condition. if (CondBOp->getOpcode() == BO_LAnd) { + MCDCLogOpStack.push_back(CondBOp); + // If we have "1 && X", simplify the code. "0 && X" would have constant // folded if the case was simple enough. bool ConstantBool = false; @@ -1735,8 +1749,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, ConstantBool) { // br(1 && X) -> br(X). incrementProfileCounter(CondBOp); - return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock, - FalseBlock, TrueCount, LH); + EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock, + FalseBlock, TrueCount, LH); + MCDCLogOpStack.pop_back(); + return; } // If we have "X && 1", simplify the code to use an uncond branch. @@ -1744,8 +1760,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) && ConstantBool) { // br(X && 1) -> br(X). - return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock, - FalseBlock, TrueCount, LH, CondBOp); + EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock, + FalseBlock, TrueCount, LH, CondBOp); + MCDCLogOpStack.pop_back(); + return; } // Emit the LHS as a conditional. If the LHS conditional is false, we @@ -1774,11 +1792,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock, FalseBlock, TrueCount, LH); eval.end(*this); - + MCDCLogOpStack.pop_back(); return; } if (CondBOp->getOpcode() == BO_LOr) { + MCDCLogOpStack.push_back(CondBOp); + // If we have "0 || X", simplify the code. "1 || X" would have constant // folded if the case was simple enough. bool ConstantBool = false; @@ -1786,8 +1806,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, !ConstantBool) { // br(0 || X) -> br(X). incrementProfileCounter(CondBOp); - return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, - FalseBlock, TrueCount, LH); + EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, + FalseBlock, TrueCount, LH); + MCDCLogOpStack.pop_back(); + return; } // If we have "X || 0", simplify the code to use an uncond branch. @@ -1795,10 +1817,11 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) && !ConstantBool) { // br(X || 0) -> br(X). - return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock, - FalseBlock, TrueCount, LH, CondBOp); + EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock, + FalseBlock, TrueCount, LH, CondBOp); + MCDCLogOpStack.pop_back(); + return; } - // Emit the LHS as a conditional. If the LHS conditional is true, we // want to jump to the TrueBlock. llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false"); @@ -1829,14 +1852,20 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, RHSCount, LH); eval.end(*this); - + MCDCLogOpStack.pop_back(); return; } } if (const UnaryOperator *CondUOp = dyn_cast(Cond)) { // br(!x, t, f) -> br(x, f, t) - if (CondUOp->getOpcode() == UO_LNot) { + // Avoid doing this optimization when instrumenting a condition for MC/DC. + // LNot is taken as part of the condition for simplicity, and changing its + // sense negatively impacts test vector tracking. + bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() && + CGM.getCodeGenOpts().MCDCCoverage && + isInstrumentedCondition(Cond); + if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) { // Negate the count. uint64_t FalseCount = getCurrentProfileCount() - TrueCount; // The values of the enum are chosen to make this negation possible. @@ -1876,14 +1905,14 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, { ApplyDebugLocation DL(*this, Cond); EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock, - LHSScaledTrueCount, LH); + LHSScaledTrueCount, LH, CondOp); } cond.end(*this); cond.begin(*this); EmitBlock(RHSBlock); EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock, - TrueCount - LHSScaledTrueCount, LH); + TrueCount - LHSScaledTrueCount, LH, CondOp); cond.end(*this); return; @@ -1906,6 +1935,21 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, CondV = EvaluateExprAsBool(Cond); } + // If not at the top of the logical operator nest, update MCDC temp with the + // boolean result of the evaluated condition. + if (!MCDCLogOpStack.empty()) { + const Expr *MCDCBaseExpr = Cond; + // When a nested ConditionalOperator (ternary) is encountered in a boolean + // expression, MC/DC tracks the result of the ternary, and this is tied to + // the ConditionalOperator expression and not the ternary's LHS or RHS. If + // this is the case, the ConditionalOperator expression is passed through + // the ConditionalOp parameter and then used as the MCDC base expression. + if (ConditionalOp) + MCDCBaseExpr = ConditionalOp; + + maybeUpdateMCDCCondBitmap(MCDCBaseExpr, CondV); + } + llvm::MDNode *Weights = nullptr; llvm::MDNode *Unpredictable = nullptr; diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 751d8110b13d..07c7678df87e 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -287,6 +287,9 @@ public: /// nest would extend. SmallVector OMPLoopNestStack; + /// Stack to track the Logical Operator recursion nest for MC/DC. + SmallVector MCDCLogOpStack; + /// Number of nested loop to be consumed by the last surrounding /// loop-associated directive. int ExpectedOMPLoopDepth = 0; @@ -1521,6 +1524,9 @@ private: CodeGenPGO PGO; + /// Bitmap used by MC/DC to track condition outcomes of a boolean expression. + Address MCDCCondBitmapAddr = Address::invalid(); + /// Calculate branch weights appropriate for PGO data llvm::MDNode *createProfileWeights(uint64_t TrueCount, uint64_t FalseCount) const; @@ -1539,6 +1545,52 @@ public: PGO.setCurrentStmt(S); } + bool isMCDCCoverageEnabled() const { + return (CGM.getCodeGenOpts().hasProfileClangInstr() && + CGM.getCodeGenOpts().MCDCCoverage && + !CurFn->hasFnAttribute(llvm::Attribute::NoProfile)); + } + + /// Allocate a temp value on the stack that MCDC can use to track condition + /// results. + void maybeCreateMCDCCondBitmap() { + if (isMCDCCoverageEnabled()) { + PGO.emitMCDCParameters(Builder); + MCDCCondBitmapAddr = + CreateIRTemp(getContext().UnsignedIntTy, "mcdc.addr"); + } + } + + bool isBinaryLogicalOp(const Expr *E) const { + const BinaryOperator *BOp = dyn_cast(E->IgnoreParens()); + return (BOp && BOp->isLogicalOp()); + } + + /// Zero-init the MCDC temp value. + void maybeResetMCDCCondBitmap(const Expr *E) { + if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) { + PGO.emitMCDCCondBitmapReset(Builder, E, MCDCCondBitmapAddr); + PGO.setCurrentStmt(E); + } + } + + /// Increment the profiler's counter for the given expression by \p StepV. + /// If \p StepV is null, the default increment is 1. + void maybeUpdateMCDCTestVectorBitmap(const Expr *E) { + if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) { + PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr); + PGO.setCurrentStmt(E); + } + } + + /// Update the MCDC temp value with the condition's evaluated result. + void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) { + if (isMCDCCoverageEnabled()) { + PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val); + PGO.setCurrentStmt(E); + } + } + /// Get the profiler's count for the given statement. uint64_t getProfileCount(const Stmt *S) { return PGO.getStmtCount(S).value_or(0); @@ -4626,6 +4678,9 @@ public: bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result, bool AllowLabels = false); + /// Ignore parentheses and logical-NOT to track conditions consistently. + static const Expr *stripCond(const Expr *C); + /// isInstrumentedCondition - Determine whether the given condition is an /// instrumentable condition (i.e. no "&&" or "||"). static bool isInstrumentedCondition(const Expr *C); @@ -4648,7 +4703,8 @@ public: /// evaluate to true based on PGO data. void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount, - Stmt::Likelihood LH = Stmt::LH_None); + Stmt::Likelihood LH = Stmt::LH_None, + const Expr *ConditionalOp = nullptr); /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is /// nonnull, if \p LHS is marked _Nonnull. diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp index 81bf8ea696b1..d68844d476eb 100644 --- a/clang/lib/CodeGen/CodeGenPGO.cpp +++ b/clang/lib/CodeGen/CodeGenPGO.cpp @@ -161,13 +161,24 @@ struct MapRegionCounters : public RecursiveASTVisitor { PGOHash Hash; /// The map of statements to counters. llvm::DenseMap &CounterMap; + /// The next bitmap byte index to assign. + unsigned NextMCDCBitmapIdx; + /// The map of statements to MC/DC bitmap coverage objects. + llvm::DenseMap &MCDCBitmapMap; + /// Maximum number of supported MC/DC conditions in a boolean expression. + unsigned MCDCMaxCond; /// The profile version. uint64_t ProfileVersion; + /// Diagnostics Engine used to report warnings. + DiagnosticsEngine &Diag; MapRegionCounters(PGOHashVersion HashVersion, uint64_t ProfileVersion, - llvm::DenseMap &CounterMap) + llvm::DenseMap &CounterMap, + llvm::DenseMap &MCDCBitmapMap, + unsigned MCDCMaxCond, DiagnosticsEngine &Diag) : NextCounter(0), Hash(HashVersion), CounterMap(CounterMap), - ProfileVersion(ProfileVersion) {} + NextMCDCBitmapIdx(0), MCDCBitmapMap(MCDCBitmapMap), + MCDCMaxCond(MCDCMaxCond), ProfileVersion(ProfileVersion), Diag(Diag) {} // Blocks and lambdas are handled as separate functions, so we need not // traverse them in the parent context. @@ -207,15 +218,126 @@ struct MapRegionCounters : public RecursiveASTVisitor { return Type; } + /// The following stacks are used with dataTraverseStmtPre() and + /// dataTraverseStmtPost() to track the depth of nested logical operators in a + /// boolean expression in a function. The ultimate purpose is to keep track + /// of the number of leaf-level conditions in the boolean expression so that a + /// profile bitmap can be allocated based on that number. + /// + /// The stacks are also used to find error cases and notify the user. A + /// standard logical operator nest for a boolean expression could be in a form + /// similar to this: "x = a && b && c && (d || f)" + unsigned NumCond = 0; + bool SplitNestedLogicalOp = false; + SmallVector NonLogOpStack; + SmallVector LogOpStack; + + // Hook: dataTraverseStmtPre() is invoked prior to visiting an AST Stmt node. + bool dataTraverseStmtPre(Stmt *S) { + /// If MC/DC is not enabled, MCDCMaxCond will be set to 0. Do nothing. + if (MCDCMaxCond == 0) + return true; + + /// At the top of the logical operator nest, reset the number of conditions. + if (LogOpStack.empty()) + NumCond = 0; + + if (const Expr *E = dyn_cast(S)) { + const BinaryOperator *BinOp = dyn_cast(E->IgnoreParens()); + if (BinOp && BinOp->isLogicalOp()) { + /// Check for "split-nested" logical operators. This happens when a new + /// boolean expression logical-op nest is encountered within an existing + /// boolean expression, separated by a non-logical operator. For + /// example, in "x = (a && b && c && foo(d && f))", the "d && f" case + /// starts a new boolean expression that is separated from the other + /// conditions by the operator foo(). Split-nested cases are not + /// supported by MC/DC. + SplitNestedLogicalOp = SplitNestedLogicalOp || !NonLogOpStack.empty(); + + LogOpStack.push_back(BinOp); + return true; + } + } + + /// Keep track of non-logical operators. These are OK as long as we don't + /// encounter a new logical operator after seeing one. + if (!LogOpStack.empty()) + NonLogOpStack.push_back(S); + + return true; + } + + // Hook: dataTraverseStmtPost() is invoked by the AST visitor after visiting + // an AST Stmt node. MC/DC will use it to to signal when the top of a + // logical operation (boolean expression) nest is encountered. + bool dataTraverseStmtPost(Stmt *S) { + /// If MC/DC is not enabled, MCDCMaxCond will be set to 0. Do nothing. + if (MCDCMaxCond == 0) + return true; + + if (const Expr *E = dyn_cast(S)) { + const BinaryOperator *BinOp = dyn_cast(E->IgnoreParens()); + if (BinOp && BinOp->isLogicalOp()) { + assert(LogOpStack.back() == BinOp); + LogOpStack.pop_back(); + + /// At the top of logical operator nest: + if (LogOpStack.empty()) { + /// Was the "split-nested" logical operator case encountered? + if (SplitNestedLogicalOp) { + unsigned DiagID = Diag.getCustomDiagID( + DiagnosticsEngine::Warning, + "unsupported MC/DC boolean expression; " + "contains an operation with a nested boolean expression. " + "Expression will not be covered"); + Diag.Report(S->getBeginLoc(), DiagID); + return false; + } + + /// Was the maximum number of conditions encountered? + if (NumCond > MCDCMaxCond) { + unsigned DiagID = Diag.getCustomDiagID( + DiagnosticsEngine::Warning, + "unsupported MC/DC boolean expression; " + "number of conditions (%0) exceeds max (%1). " + "Expression will not be covered"); + Diag.Report(S->getBeginLoc(), DiagID) << NumCond << MCDCMaxCond; + return false; + } + + // Otherwise, allocate the number of bytes required for the bitmap + // based on the number of conditions. Must be at least 1-byte long. + MCDCBitmapMap[BinOp] = NextMCDCBitmapIdx; + unsigned SizeInBits = std::max(1L << NumCond, CHAR_BIT); + NextMCDCBitmapIdx += SizeInBits / CHAR_BIT; + } + return true; + } + } + + if (!LogOpStack.empty()) + NonLogOpStack.pop_back(); + + return true; + } + /// The RHS of all logical operators gets a fresh counter in order to count /// how many times the RHS evaluates to true or false, depending on the /// semantics of the operator. This is only valid for ">= v7" of the profile - /// version so that we facilitate backward compatibility. + /// version so that we facilitate backward compatibility. In addition, in + /// order to use MC/DC, count the number of total LHS and RHS conditions. bool VisitBinaryOperator(BinaryOperator *S) { - if (ProfileVersion >= llvm::IndexedInstrProf::Version7) - if (S->isLogicalOp() && - CodeGenFunction::isInstrumentedCondition(S->getRHS())) - CounterMap[S->getRHS()] = NextCounter++; + if (S->isLogicalOp()) { + if (CodeGenFunction::isInstrumentedCondition(S->getLHS())) + NumCond++; + + if (CodeGenFunction::isInstrumentedCondition(S->getRHS())) { + if (ProfileVersion >= llvm::IndexedInstrProf::Version7) + CounterMap[S->getRHS()] = NextCounter++; + + NumCond++; + } + } return Base::VisitBinaryOperator(S); } @@ -851,8 +973,22 @@ void CodeGenPGO::mapRegionCounters(const Decl *D) { ProfileVersion = PGOReader->getVersion(); } + // If MC/DC is enabled, set the MaxConditions to a preset value. Otherwise, + // set it to zero. This value impacts the number of conditions accepted in a + // given boolean expression, which impacts the size of the bitmap used to + // track test vector execution for that boolean expression. Because the + // bitmap scales exponentially (2^n) based on the number of conditions seen, + // the maximum value is hard-coded at 6 conditions, which is more than enough + // for most embedded applications. Setting a maximum value prevents the + // bitmap footprint from growing too large without the user's knowledge. In + // the future, this value could be adjusted with a command-line option. + unsigned MCDCMaxConditions = (CGM.getCodeGenOpts().MCDCCoverage) ? 6 : 0; + RegionCounterMap.reset(new llvm::DenseMap); - MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap); + RegionMCDCBitmapMap.reset(new llvm::DenseMap); + MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap, + *RegionMCDCBitmapMap, MCDCMaxConditions, + CGM.getDiags()); if (const FunctionDecl *FD = dyn_cast_or_null(D)) Walker.TraverseDecl(const_cast(FD)); else if (const ObjCMethodDecl *MD = dyn_cast_or_null(D)) @@ -863,6 +999,7 @@ void CodeGenPGO::mapRegionCounters(const Decl *D) { Walker.TraverseDecl(const_cast(CD)); assert(Walker.NextCounter > 0 && "no entry counter mapped for decl"); NumRegionCounters = Walker.NextCounter; + MCDCBitmapBytes = Walker.NextMCDCBitmapIdx; FunctionHash = Walker.Hash.finalize(); } @@ -894,9 +1031,11 @@ void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { std::string CoverageMapping; llvm::raw_string_ostream OS(CoverageMapping); - CoverageMappingGen MappingGen(*CGM.getCoverageMapping(), - CGM.getContext().getSourceManager(), - CGM.getLangOpts(), RegionCounterMap.get()); + RegionCondIDMap.reset(new llvm::DenseMap); + CoverageMappingGen MappingGen( + *CGM.getCoverageMapping(), CGM.getContext().getSourceManager(), + CGM.getLangOpts(), RegionCounterMap.get(), RegionMCDCBitmapMap.get(), + RegionCondIDMap.get()); MappingGen.emitCounterMapping(D, OS); OS.flush(); @@ -972,6 +1111,108 @@ void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S, ArrayRef(Args)); } +bool CodeGenPGO::canEmitMCDCCoverage(const CGBuilderTy &Builder) { + return (CGM.getCodeGenOpts().hasProfileClangInstr() && + CGM.getCodeGenOpts().MCDCCoverage && Builder.GetInsertBlock()); +} + +void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) { + if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap) + return; + + auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext()); + + // Emit intrinsic representing MCDC bitmap parameters at function entry. + // This is used by the instrumentation pass, but it isn't actually lowered to + // anything. + llvm::Value *Args[3] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + Builder.getInt64(FunctionHash), + Builder.getInt32(MCDCBitmapBytes)}; + Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_parameters), Args); +} + +void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, + const Expr *S, + Address MCDCCondBitmapAddr) { + if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap) + return; + + S = S->IgnoreParens(); + + auto ExprMCDCBitmapMapIterator = RegionMCDCBitmapMap->find(S); + if (ExprMCDCBitmapMapIterator == RegionMCDCBitmapMap->end()) + return; + + // Extract the ID of the global bitmap associated with this expression. + unsigned MCDCTestVectorBitmapID = ExprMCDCBitmapMapIterator->second; + auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext()); + + // Emit intrinsic responsible for updating the global bitmap corresponding to + // a boolean expression. The index being set is based on the value loaded + // from a pointer to a dedicated temporary value on the stack that is itself + // updated via emitMCDCCondBitmapReset() and emitMCDCCondBitmapUpdate(). The + // index represents an executed test vector. + llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + Builder.getInt64(FunctionHash), + Builder.getInt32(MCDCBitmapBytes), + Builder.getInt32(MCDCTestVectorBitmapID), + MCDCCondBitmapAddr.getPointer()}; + Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args); +} + +void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, + Address MCDCCondBitmapAddr) { + if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap) + return; + + S = S->IgnoreParens(); + + if (RegionMCDCBitmapMap->find(S) == RegionMCDCBitmapMap->end()) + return; + + // Emit intrinsic that resets a dedicated temporary value on the stack to 0. + Builder.CreateStore(Builder.getInt32(0), MCDCCondBitmapAddr); +} + +void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, + Address MCDCCondBitmapAddr, + llvm::Value *Val) { + if (!canEmitMCDCCoverage(Builder) || !RegionCondIDMap) + return; + + // Even though, for simplicity, parentheses and unary logical-NOT operators + // are considered part of their underlying condition for both MC/DC and + // branch coverage, the condition IDs themselves are assigned and tracked + // using the underlying condition itself. This is done solely for + // consistency since parentheses and logical-NOTs are ignored when checking + // whether the condition is actually an instrumentable condition. This can + // also make debugging a bit easier. + S = CodeGenFunction::stripCond(S); + + auto ExprMCDCConditionIDMapIterator = RegionCondIDMap->find(S); + if (ExprMCDCConditionIDMapIterator == RegionCondIDMap->end()) + return; + + // Extract the ID of the condition we are setting in the bitmap. + unsigned CondID = ExprMCDCConditionIDMapIterator->second; + assert(CondID > 0 && "Condition has no ID!"); + + auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext()); + + // Emit intrinsic that updates a dedicated temporary value on the stack after + // a condition is evaluated. After the set of conditions has been updated, + // the resulting value is used to update the boolean expression's bitmap. + llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + Builder.getInt64(FunctionHash), + Builder.getInt32(CondID - 1), + MCDCCondBitmapAddr.getPointer(), Val}; + Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update), + Args); +} + void CodeGenPGO::setValueProfilingFlag(llvm::Module &M) { if (CGM.getCodeGenOpts().hasProfileClangInstr()) M.addModuleFlag(llvm::Module::Warning, "EnableValueProfiling", diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h index 392ec5a144fe..6596b6c35277 100644 --- a/clang/lib/CodeGen/CodeGenPGO.h +++ b/clang/lib/CodeGen/CodeGenPGO.h @@ -33,8 +33,11 @@ private: std::array NumValueSites; unsigned NumRegionCounters; + unsigned MCDCBitmapBytes; uint64_t FunctionHash; std::unique_ptr> RegionCounterMap; + std::unique_ptr> RegionMCDCBitmapMap; + std::unique_ptr> RegionCondIDMap; std::unique_ptr> StmtCountMap; std::unique_ptr ProfRecord; std::vector RegionCounts; @@ -43,7 +46,8 @@ private: public: CodeGenPGO(CodeGenModule &CGModule) : CGM(CGModule), FuncNameVar(nullptr), NumValueSites({{0}}), - NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0) {} + NumRegionCounters(0), MCDCBitmapBytes(0), FunctionHash(0), + CurrentRegionCount(0) {} /// Whether or not we have PGO region data for the current function. This is /// false both when we have no data at all and when our data has been @@ -103,10 +107,18 @@ private: bool IsInMainFile); bool skipRegionMappingForDecl(const Decl *D); void emitCounterRegionMapping(const Decl *D); + bool canEmitMCDCCoverage(const CGBuilderTy &Builder); public: void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S, llvm::Value *StepV); + void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S, + Address MCDCCondBitmapAddr); + void emitMCDCParameters(CGBuilderTy &Builder); + void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S, + Address MCDCCondBitmapAddr); + void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S, + Address MCDCCondBitmapAddr, llvm::Value *Val); /// Return the region count for the counter at the given index. uint64_t getRegionCount(const Stmt *S) { diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp index 56411e2240e5..bf227386a71b 100644 --- a/clang/lib/CodeGen/CoverageMappingGen.cpp +++ b/clang/lib/CodeGen/CoverageMappingGen.cpp @@ -95,6 +95,8 @@ void CoverageSourceInfo::updateNextTokLoc(SourceLocation Loc) { } namespace { +using MCDCConditionID = CounterMappingRegion::MCDCConditionID; +using MCDCParameters = CounterMappingRegion::MCDCParameters; /// A region of source code that can be mapped to a counter. class SourceMappingRegion { @@ -104,6 +106,9 @@ class SourceMappingRegion { /// Secondary Counter used for Branch Regions for "False" branches. std::optional FalseCount; + /// Parameters used for Modified Condition/Decision Coverage + MCDCParameters MCDCParams; + /// The region's starting location. std::optional LocStart; @@ -122,11 +127,18 @@ public: } SourceMappingRegion(Counter Count, std::optional FalseCount, + MCDCParameters MCDCParams, std::optional LocStart, std::optional LocEnd, bool GapRegion = false) - : Count(Count), FalseCount(FalseCount), LocStart(LocStart), - LocEnd(LocEnd), GapRegion(GapRegion) {} + : Count(Count), FalseCount(FalseCount), MCDCParams(MCDCParams), + LocStart(LocStart), LocEnd(LocEnd), GapRegion(GapRegion) {} + + SourceMappingRegion(MCDCParameters MCDCParams, + std::optional LocStart, + std::optional LocEnd) + : MCDCParams(MCDCParams), LocStart(LocStart), LocEnd(LocEnd), + GapRegion(false) {} const Counter &getCounter() const { return Count; } @@ -163,6 +175,10 @@ public: void setGap(bool Gap) { GapRegion = Gap; } bool isBranch() const { return FalseCount.has_value(); } + + bool isMCDCDecision() const { return MCDCParams.NumConditions != 0; } + + const MCDCParameters &getMCDCParams() const { return MCDCParams; } }; /// Spelling locations for the start and end of a source region. @@ -454,8 +470,13 @@ public: SR.LineEnd, SR.ColumnEnd)); } else if (Region.isBranch()) { MappingRegions.push_back(CounterMappingRegion::makeBranchRegion( - Region.getCounter(), Region.getFalseCounter(), *CovFileID, - SR.LineStart, SR.ColumnStart, SR.LineEnd, SR.ColumnEnd)); + Region.getCounter(), Region.getFalseCounter(), + Region.getMCDCParams(), *CovFileID, SR.LineStart, SR.ColumnStart, + SR.LineEnd, SR.ColumnEnd)); + } else if (Region.isMCDCDecision()) { + MappingRegions.push_back(CounterMappingRegion::makeDecisionRegion( + Region.getMCDCParams(), *CovFileID, SR.LineStart, SR.ColumnStart, + SR.LineEnd, SR.ColumnEnd)); } else { MappingRegions.push_back(CounterMappingRegion::makeRegion( Region.getCounter(), *CovFileID, SR.LineStart, SR.ColumnStart, @@ -542,6 +563,239 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder { } }; +/// A wrapper object for maintaining stacks to track the resursive AST visitor +/// walks for the purpose of assigning IDs to leaf-level conditions measured by +/// MC/DC. The object is created with a reference to the MCDCBitmapMap that was +/// created during the initial AST walk. The presence of a bitmap associated +/// with a boolean expression (top-level logical operator nest) indicates that +/// the boolean expression qualified for MC/DC. The resulting condition IDs +/// are preserved in a map reference that is also provided during object +/// creation. +struct MCDCCoverageBuilder { + + /// The AST walk recursively visits nested logical-AND or logical-OR binary + /// operator nodes and then visits their LHS and RHS children nodes. As this + /// happens, the algorithm will assign IDs to each operator's LHS and RHS side + /// as the walk moves deeper into the nest. At each level of the recursive + /// nest, the LHS and RHS may actually correspond to larger subtrees (not + /// leaf-conditions). If this is the case, when that node is visited, the ID + /// assigned to the subtree is re-assigned to its LHS, and a new ID is given + /// to its RHS. At the end of the walk, all leaf-level conditions will have a + /// unique ID -- keep in mind that the final set of IDs may not be in + /// numerical order from left to right. + /// + /// Example: "x = (A && B) || (C && D) || (D && F)" + /// + /// Visit Depth1: + /// (A && B) || (C && D) || (D && F) + /// ^-------LHS--------^ ^-RHS--^ + /// ID=1 ID=2 + /// + /// Visit LHS-Depth2: + /// (A && B) || (C && D) + /// ^-LHS--^ ^-RHS--^ + /// ID=1 ID=3 + /// + /// Visit LHS-Depth3: + /// (A && B) + /// LHS RHS + /// ID=1 ID=4 + /// + /// Visit RHS-Depth3: + /// (C && D) + /// LHS RHS + /// ID=3 ID=5 + /// + /// Visit RHS-Depth2: (D && F) + /// LHS RHS + /// ID=2 ID=6 + /// + /// Visit Depth1: + /// (A && B) || (C && D) || (D && F) + /// ID=1 ID=4 ID=3 ID=5 ID=2 ID=6 + /// + /// A node ID of '0' always means MC/DC isn't being tracked. + /// + /// As the AST walk proceeds recursively, the algorithm will also use stacks + /// to track the IDs of logical-AND and logical-OR operations on the RHS so + /// that it can be determined which nodes are executed next, depending on how + /// a LHS or RHS of a logical-AND or logical-OR is evaluated. This + /// information relies on the assigned IDs and are embedded within the + /// coverage region IDs of each branch region associated with a leaf-level + /// condition. This information helps the visualization tool reconstruct all + /// possible test vectors for the purposes of MC/DC analysis. if a "next" node + /// ID is '0', it means it's the end of the test vector. The following rules + /// are used: + /// + /// For logical-AND ("LHS && RHS"): + /// - If LHS is TRUE, execution goes to the RHS node. + /// - If LHS is FALSE, execution goes to the LHS node of the next logical-OR. + /// If that does not exist, execution exits (ID == 0). + /// + /// - If RHS is TRUE, execution goes to LHS node of the next logical-AND. + /// If that does not exist, execution exits (ID == 0). + /// - If RHS is FALSE, execution goes to the LHS node of the next logical-OR. + /// If that does not exist, execution exits (ID == 0). + /// + /// For logical-OR ("LHS || RHS"): + /// - If LHS is TRUE, execution goes to the LHS node of the next logical-AND. + /// If that does not exist, execution exits (ID == 0). + /// - If LHS is FALSE, execution goes to the RHS node. + /// + /// - If RHS is TRUE, execution goes to LHS node of the next logical-AND. + /// If that does not exist, execution exits (ID == 0). + /// - If RHS is FALSE, execution goes to the LHS node of the next logical-OR. + /// If that does not exist, execution exits (ID == 0). + /// + /// Finally, the condition IDs are also used when instrumenting the code to + /// indicate a unique offset into a temporary bitmap that represents the true + /// or false evaluation of that particular condition. + /// + /// NOTE regarding the use of CodeGenFunction::stripCond(). Even though, for + /// simplicity, parentheses and unary logical-NOT operators are considered + /// part of their underlying condition for both MC/DC and branch coverage, the + /// condition IDs themselves are assigned and tracked using the underlying + /// condition itself. This is done solely for consistency since parentheses + /// and logical-NOTs are ignored when checking whether the condition is + /// actually an instrumentable condition. This can also make debugging a bit + /// easier. + +private: + CodeGenModule &CGM; + + llvm::SmallVector AndRHS; + llvm::SmallVector OrRHS; + llvm::SmallVector NestLevel; + llvm::DenseMap &CondIDs; + llvm::DenseMap &MCDCBitmapMap; + MCDCConditionID NextID = 1; + bool NotMapped = false; + + /// Is this a logical-AND operation? + bool isLAnd(const BinaryOperator *E) const { + return E->getOpcode() == BO_LAnd; + } + + /// Push an ID onto the corresponding RHS stack. + void pushRHS(const BinaryOperator *E) { + llvm::SmallVector &rhs = isLAnd(E) ? AndRHS : OrRHS; + rhs.push_back(CondIDs[CodeGenFunction::stripCond(E->getRHS())]); + } + + /// Pop an ID from the corresponding RHS stack. + void popRHS(const BinaryOperator *E) { + llvm::SmallVector &rhs = isLAnd(E) ? AndRHS : OrRHS; + if (!rhs.empty()) + rhs.pop_back(); + } + + /// If the expected ID is on top, pop it off the corresponding RHS stack. + void popRHSifTop(const BinaryOperator *E) { + if (!OrRHS.empty() && CondIDs[E] == OrRHS.back()) + OrRHS.pop_back(); + else if (!AndRHS.empty() && CondIDs[E] == AndRHS.back()) + AndRHS.pop_back(); + } + +public: + MCDCCoverageBuilder(CodeGenModule &CGM, + llvm::DenseMap &CondIDMap, + llvm::DenseMap &MCDCBitmapMap) + : CGM(CGM), CondIDs(CondIDMap), MCDCBitmapMap(MCDCBitmapMap) {} + + /// Return the ID of the RHS of the next, upper nest-level logical-OR. + MCDCConditionID getNextLOrCondID() const { + return OrRHS.empty() ? 0 : OrRHS.back(); + } + + /// Return the ID of the RHS of the next, upper nest-level logical-AND. + MCDCConditionID getNextLAndCondID() const { + return AndRHS.empty() ? 0 : AndRHS.back(); + } + + /// Return the ID of a given condition. + MCDCConditionID getCondID(const Expr *Cond) const { + auto I = CondIDs.find(CodeGenFunction::stripCond(Cond)); + if (I == CondIDs.end()) + return 0; + else + return I->second; + } + + /// Push the binary operator statement to track the nest level and assign IDs + /// to the operator's LHS and RHS. The RHS may be a larger subtree that is + /// broken up on successive levels. + void pushAndAssignIDs(const BinaryOperator *E) { + if (!CGM.getCodeGenOpts().MCDCCoverage) + return; + + // If binary expression is disqualified, don't do mapping. + if (NestLevel.empty() && MCDCBitmapMap.find(CodeGenFunction::stripCond( + E)) == MCDCBitmapMap.end()) + NotMapped = true; + + // Push Stmt on 'NestLevel' stack to keep track of nest location. + NestLevel.push_back(E); + + // Don't go any further if we don't need to map condition IDs. + if (NotMapped) + return; + + // If the operator itself has an assigned ID, this means it represents a + // larger subtree. In this case, pop its ID out of the RHS stack and + // assign that ID to its LHS node. Its RHS will receive a new ID. + if (CondIDs.find(CodeGenFunction::stripCond(E)) != CondIDs.end()) { + // If Stmt has an ID, assign its ID to LHS + CondIDs[CodeGenFunction::stripCond(E->getLHS())] = CondIDs[E]; + + // Since the operator's LHS assumes the operator's same ID, pop the + // operator from the RHS stack so that if LHS short-circuits, it won't be + // incorrectly re-used as the node executed next. + popRHSifTop(E); + } else { + // Otherwise, assign ID+1 to LHS. + CondIDs[CodeGenFunction::stripCond(E->getLHS())] = NextID++; + } + + // Assign ID+1 to RHS. + CondIDs[CodeGenFunction::stripCond(E->getRHS())] = NextID++; + + // Push ID of Stmt's RHS so that LHS nodes know about it + pushRHS(E); + } + + /// Pop the binary operator from the next level. If the walk is at the top of + /// the next, assign the total number of conditions. + unsigned popAndReturnCondCount(const BinaryOperator *E) { + if (!CGM.getCodeGenOpts().MCDCCoverage) + return 0; + + unsigned TotalConds = 0; + + // Pop Stmt from 'NestLevel' stack. + assert(NestLevel.back() == E); + NestLevel.pop_back(); + + // Reset state if not doing mapping. + if (NestLevel.empty() && NotMapped) { + NotMapped = false; + return 0; + } + + // Pop RHS ID. + popRHS(E); + + // If at the parent (NestLevel=0), set conds and reset. + if (NestLevel.empty()) { + TotalConds = NextID - 1; + + // Reset ID back to beginning. + NextID = 1; + } + return TotalConds; + } +}; + /// A StmtVisitor that creates coverage mapping regions which map /// from the source code locations to the PGO counters. struct CounterCoverageMappingBuilder @@ -550,8 +804,14 @@ struct CounterCoverageMappingBuilder /// The map of statements to count values. llvm::DenseMap &CounterMap; + /// The map of statements to bitmap coverage object values. + llvm::DenseMap &MCDCBitmapMap; + /// A stack of currently live regions. - std::vector RegionStack; + llvm::SmallVector RegionStack; + + /// An object to manage MCDC regions. + MCDCCoverageBuilder MCDCBuilder; CounterExpressionBuilder Builder; @@ -589,6 +849,8 @@ struct CounterCoverageMappingBuilder return Counter::getCounter(CounterMap[S]); } + unsigned getRegionBitmap(const Stmt *S) { return MCDCBitmapMap[S]; } + /// Push a region onto the stack. /// /// Returns the index on the stack where the region was pushed. This can be @@ -596,7 +858,9 @@ struct CounterCoverageMappingBuilder size_t pushRegion(Counter Count, std::optional StartLoc = std::nullopt, std::optional EndLoc = std::nullopt, - std::optional FalseCount = std::nullopt) { + std::optional FalseCount = std::nullopt, + MCDCConditionID ID = 0, MCDCConditionID TrueID = 0, + MCDCConditionID FalseID = 0) { if (StartLoc && !FalseCount) { MostRecentLocation = *StartLoc; @@ -615,7 +879,19 @@ struct CounterCoverageMappingBuilder StartLoc = std::nullopt; if (EndLoc && EndLoc->isInvalid()) EndLoc = std::nullopt; - RegionStack.emplace_back(Count, FalseCount, StartLoc, EndLoc); + RegionStack.emplace_back(Count, FalseCount, + MCDCParameters{0, 0, ID, TrueID, FalseID}, + StartLoc, EndLoc); + + return RegionStack.size() - 1; + } + + size_t pushRegion(unsigned BitmapIdx, unsigned Conditions, + std::optional StartLoc = std::nullopt, + std::optional EndLoc = std::nullopt) { + + RegionStack.emplace_back(MCDCParameters{BitmapIdx, Conditions}, StartLoc, + EndLoc); return RegionStack.size() - 1; } @@ -746,7 +1022,9 @@ struct CounterCoverageMappingBuilder /// and add it to the function's SourceRegions. A branch region tracks a /// "True" counter and a "False" counter for boolean expressions that /// result in the generation of a branch. - void createBranchRegion(const Expr *C, Counter TrueCnt, Counter FalseCnt) { + void createBranchRegion(const Expr *C, Counter TrueCnt, Counter FalseCnt, + MCDCConditionID ID = 0, MCDCConditionID TrueID = 0, + MCDCConditionID FalseID = 0) { // Check for NULL conditions. if (!C) return; @@ -764,13 +1042,21 @@ struct CounterCoverageMappingBuilder // CodeGenFunction.c always returns false, but that is very heavy-handed. if (ConditionFoldsToBool(C)) popRegions(pushRegion(Counter::getZero(), getStart(C), getEnd(C), - Counter::getZero())); + Counter::getZero(), ID, TrueID, FalseID)); else // Otherwise, create a region with the True counter and False counter. - popRegions(pushRegion(TrueCnt, getStart(C), getEnd(C), FalseCnt)); + popRegions(pushRegion(TrueCnt, getStart(C), getEnd(C), FalseCnt, ID, + TrueID, FalseID)); } } + /// Create a Decision Region with a BitmapIdx and number of Conditions. This + /// type of region "contains" branch regions, one for each of the conditions. + /// The visualization tool will group everything together. + void createDecisionRegion(const Expr *C, unsigned BitmapIdx, unsigned Conds) { + popRegions(pushRegion(BitmapIdx, Conds, getStart(C), getEnd(C))); + } + /// Create a Branch Region around a SwitchCase for code coverage /// and add it to the function's SourceRegions. void createSwitchCaseRegion(const SwitchCase *SC, Counter TrueCnt, @@ -851,8 +1137,12 @@ struct CounterCoverageMappingBuilder // we've seen this region. if (StartLocs.insert(Loc).second) { if (I.isBranch()) - SourceRegions.emplace_back(I.getCounter(), I.getFalseCounter(), Loc, - getEndOfFileOrMacro(Loc), I.isBranch()); + SourceRegions.emplace_back( + I.getCounter(), I.getFalseCounter(), + MCDCParameters{0, 0, I.getMCDCParams().ID, + I.getMCDCParams().TrueID, + I.getMCDCParams().FalseID}, + Loc, getEndOfFileOrMacro(Loc), I.isBranch()); else SourceRegions.emplace_back(I.getCounter(), Loc, getEndOfFileOrMacro(Loc)); @@ -971,9 +1261,13 @@ struct CounterCoverageMappingBuilder CounterCoverageMappingBuilder( CoverageMappingModuleGen &CVM, - llvm::DenseMap &CounterMap, SourceManager &SM, - const LangOptions &LangOpts) - : CoverageMappingBuilder(CVM, SM, LangOpts), CounterMap(CounterMap) {} + llvm::DenseMap &CounterMap, + llvm::DenseMap &MCDCBitmapMap, + llvm::DenseMap &CondIDMap, + SourceManager &SM, const LangOptions &LangOpts) + : CoverageMappingBuilder(CVM, SM, LangOpts), CounterMap(CounterMap), + MCDCBitmapMap(MCDCBitmapMap), + MCDCBuilder(CVM.getCodeGenModule(), CondIDMap, MCDCBitmapMap) {} /// Write the mapping data to the output stream void write(llvm::raw_ostream &OS) { @@ -1519,6 +1813,9 @@ struct CounterCoverageMappingBuilder } void VisitBinLAnd(const BinaryOperator *E) { + // Keep track of Binary Operator and assign MCDC condition IDs + MCDCBuilder.pushAndAssignIDs(E); + extendRegion(E->getLHS()); propagateCounts(getRegion().getCounter(), E->getLHS()); handleFileExit(getEnd(E->getLHS())); @@ -1527,6 +1824,11 @@ struct CounterCoverageMappingBuilder extendRegion(E->getRHS()); propagateCounts(getRegionCounter(E), E->getRHS()); + // Process Binary Operator and create MCDC Decision Region if top-level + unsigned NumConds = 0; + if ((NumConds = MCDCBuilder.popAndReturnCondCount(E))) + createDecisionRegion(E, getRegionBitmap(E), NumConds); + // Extract the RHS's Execution Counter. Counter RHSExecCnt = getRegionCounter(E); @@ -1536,13 +1838,30 @@ struct CounterCoverageMappingBuilder // Extract the Parent Region Counter. Counter ParentCnt = getRegion().getCounter(); + // Extract the MCDC condition IDs (returns 0 if not needed). + MCDCConditionID NextOrID = MCDCBuilder.getNextLOrCondID(); + MCDCConditionID NextAndID = MCDCBuilder.getNextLAndCondID(); + MCDCConditionID LHSid = MCDCBuilder.getCondID(E->getLHS()); + MCDCConditionID RHSid = MCDCBuilder.getCondID(E->getRHS()); + // Create Branch Region around LHS condition. + // MC/DC: For "LHS && RHS" + // - If LHS is TRUE, execution goes to the RHS. + // - If LHS is FALSE, execution goes to the LHS of the next logical-OR. + // If that does not exist, execution exits (ID == 0). createBranchRegion(E->getLHS(), RHSExecCnt, - subtractCounters(ParentCnt, RHSExecCnt)); + subtractCounters(ParentCnt, RHSExecCnt), LHSid, RHSid, + NextOrID); // Create Branch Region around RHS condition. + // MC/DC: For "LHS && RHS" + // - If RHS is TRUE, execution goes to LHS of the next logical-AND. + // If that does not exist, execution exits (ID == 0). + // - If RHS is FALSE, execution goes to the LHS of the next logical-OR. + // If that does not exist, execution exits (ID == 0). createBranchRegion(E->getRHS(), RHSTrueCnt, - subtractCounters(RHSExecCnt, RHSTrueCnt)); + subtractCounters(RHSExecCnt, RHSTrueCnt), RHSid, + NextAndID, NextOrID); } // Determine whether the right side of OR operation need to be visited. @@ -1556,6 +1875,9 @@ struct CounterCoverageMappingBuilder } void VisitBinLOr(const BinaryOperator *E) { + // Keep track of Binary Operator and assign MCDC condition IDs + MCDCBuilder.pushAndAssignIDs(E); + extendRegion(E->getLHS()); Counter OutCount = propagateCounts(getRegion().getCounter(), E->getLHS()); handleFileExit(getEnd(E->getLHS())); @@ -1564,6 +1886,11 @@ struct CounterCoverageMappingBuilder extendRegion(E->getRHS()); propagateCounts(getRegionCounter(E), E->getRHS()); + // Process Binary Operator and create MCDC Decision Region if top-level + unsigned NumConds = 0; + if ((NumConds = MCDCBuilder.popAndReturnCondCount(E))) + createDecisionRegion(E, getRegionBitmap(E), NumConds); + // Extract the RHS's Execution Counter. Counter RHSExecCnt = getRegionCounter(E); @@ -1577,13 +1904,28 @@ struct CounterCoverageMappingBuilder // Extract the Parent Region Counter. Counter ParentCnt = getRegion().getCounter(); + // Extract the MCDC condition IDs (returns 0 if not needed). + MCDCConditionID NextOrID = MCDCBuilder.getNextLOrCondID(); + MCDCConditionID NextAndID = MCDCBuilder.getNextLAndCondID(); + MCDCConditionID LHSid = MCDCBuilder.getCondID(E->getLHS()); + MCDCConditionID RHSid = MCDCBuilder.getCondID(E->getRHS()); + // Create Branch Region around LHS condition. + // MC/DC: For "LHS || RHS" + // - If LHS is TRUE, execution goes to the LHS of the next logical-AND. + // If that does not exist, execution exits (ID == 0). + // - If LHS is FALSE, execution goes to the RHS. createBranchRegion(E->getLHS(), subtractCounters(ParentCnt, RHSExecCnt), - RHSExecCnt); + RHSExecCnt, LHSid, NextAndID, RHSid); // Create Branch Region around RHS condition. + // MC/DC: For "LHS || RHS" + // - If RHS is TRUE, execution goes to LHS of the next logical-AND. + // If that does not exist, execution exits (ID == 0). + // - If RHS is FALSE, execution goes to the LHS of the next logical-OR. + // If that does not exist, execution exits (ID == 0). createBranchRegion(E->getRHS(), subtractCounters(RHSExecCnt, RHSFalseCnt), - RHSFalseCnt); + RHSFalseCnt, RHSid, NextAndID, NextOrID); } void VisitLambdaExpr(const LambdaExpr *LE) { @@ -1633,11 +1975,23 @@ static void dump(llvm::raw_ostream &OS, StringRef FunctionName, OS << "File " << R.FileID << ", " << R.LineStart << ":" << R.ColumnStart << " -> " << R.LineEnd << ":" << R.ColumnEnd << " = "; - Ctx.dump(R.Count, OS); - if (R.Kind == CounterMappingRegion::BranchRegion) { - OS << ", "; - Ctx.dump(R.FalseCount, OS); + if (R.Kind == CounterMappingRegion::MCDCDecisionRegion) { + OS << "M:" << R.MCDCParams.BitmapIdx; + OS << ", C:" << R.MCDCParams.NumConditions; + } else { + Ctx.dump(R.Count, OS); + + if (R.Kind == CounterMappingRegion::BranchRegion || + R.Kind == CounterMappingRegion::MCDCBranchRegion) { + OS << ", "; + Ctx.dump(R.FalseCount, OS); + } + } + + if (R.Kind == CounterMappingRegion::MCDCBranchRegion) { + OS << " [" << R.MCDCParams.ID << "," << R.MCDCParams.TrueID; + OS << "," << R.MCDCParams.FalseID << "] "; } if (R.Kind == CounterMappingRegion::ExpansionRegion) @@ -1846,8 +2200,9 @@ unsigned CoverageMappingModuleGen::getFileID(FileEntryRef File) { void CoverageMappingGen::emitCounterMapping(const Decl *D, llvm::raw_ostream &OS) { - assert(CounterMap); - CounterCoverageMappingBuilder Walker(CVM, *CounterMap, SM, LangOpts); + assert(CounterMap && MCDCBitmapMap); + CounterCoverageMappingBuilder Walker(CVM, *CounterMap, *MCDCBitmapMap, + *CondIDMap, SM, LangOpts); Walker.VisitDecl(D); Walker.write(OS); } diff --git a/clang/lib/CodeGen/CoverageMappingGen.h b/clang/lib/CodeGen/CoverageMappingGen.h index 77d7c6cd87cf..62cea173c9fc 100644 --- a/clang/lib/CodeGen/CoverageMappingGen.h +++ b/clang/lib/CodeGen/CoverageMappingGen.h @@ -150,16 +150,22 @@ class CoverageMappingGen { SourceManager &SM; const LangOptions &LangOpts; llvm::DenseMap *CounterMap; + llvm::DenseMap *MCDCBitmapMap; + llvm::DenseMap *CondIDMap; public: CoverageMappingGen(CoverageMappingModuleGen &CVM, SourceManager &SM, const LangOptions &LangOpts) - : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(nullptr) {} + : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(nullptr), + MCDCBitmapMap(nullptr), CondIDMap(nullptr) {} CoverageMappingGen(CoverageMappingModuleGen &CVM, SourceManager &SM, const LangOptions &LangOpts, - llvm::DenseMap *CounterMap) - : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(CounterMap) {} + llvm::DenseMap *CounterMap, + llvm::DenseMap *MCDCBitmapMap, + llvm::DenseMap *CondIDMap) + : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(CounterMap), + MCDCBitmapMap(MCDCBitmapMap), CondIDMap(CondIDMap) {} /// Emit the coverage mapping data which maps the regions of /// code to counters that will be used to find the execution diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index acfa11980506..2d8ef841d4f6 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -698,6 +698,17 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C, CmdArgs.push_back("-fcoverage-mapping"); } + if (Args.hasFlag(options::OPT_fmcdc_coverage, options::OPT_fno_mcdc_coverage, + false)) { + if (!Args.hasFlag(options::OPT_fcoverage_mapping, + options::OPT_fno_coverage_mapping, false)) + D.Diag(clang::diag::err_drv_argument_only_allowed_with) + << "-fcoverage-mcdc" + << "-fcoverage-mapping"; + + CmdArgs.push_back("-fcoverage-mcdc"); + } + if (Arg *A = Args.getLastArg(options::OPT_ffile_compilation_dir_EQ, options::OPT_fcoverage_compilation_dir_EQ)) { if (A->getOption().matches(options::OPT_ffile_compilation_dir_EQ)) diff --git a/clang/test/CoverageMapping/branch-constfolded.cpp b/clang/test/CoverageMapping/branch-constfolded.cpp index 5173286addbb..4fdb640506c9 100644 --- a/clang/test/CoverageMapping/branch-constfolded.cpp +++ b/clang/test/CoverageMapping/branch-constfolded.cpp @@ -1,90 +1,100 @@ // Test that branch regions are not generated for constant-folded conditions. // RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name branch-constfolded.cpp %s | FileCheck %s +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name branch-constfolded.cpp %s | FileCheck %s -check-prefix=MCDC // CHECK-LABEL: _Z6fand_0b: -bool fand_0(bool a) { +bool fand_0(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:20 = M:0, C:2 return false && a; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:15 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:19 -> [[@LINE-1]]:20 = #2, (#1 - #2) // CHECK-LABEL: _Z6fand_1b: -bool fand_1(bool a) { +bool fand_1(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:19 = M:0, C:2 return a && true; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = #1, (#0 - #1) } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:19 = 0, 0 // CHECK-LABEL: _Z6fand_2bb: -bool fand_2(bool a, bool b) { +bool fand_2(bool a, bool b) {// MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:25 = M:0, C:3 return false && a && b; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:15 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:19 -> [[@LINE-1]]:20 = #4, (#3 - #4) // CHECK: Branch,File 0, [[@LINE-2]]:24 -> [[@LINE-2]]:25 = #2, (#1 - #2) // CHECK-LABEL: _Z6fand_3bb: -bool fand_3(bool a, bool b) { +bool fand_3(bool a, bool b) {// MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:24 = M:0, C:3 return a && true && b; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = #3, (#0 - #3) } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:19 = 0, 0 // CHECK: Branch,File 0, [[@LINE-2]]:23 -> [[@LINE-2]]:24 = #2, (#1 - #2) // CHECK-LABEL: _Z6fand_4bb: -bool fand_4(bool a, bool b) { +bool fand_4(bool a, bool b) {// MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:25 = M:0, C:3 return a && b && false; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = #3, (#0 - #3) } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:16 = #4, (#3 - #4) // CHECK: Branch,File 0, [[@LINE-2]]:20 -> [[@LINE-2]]:25 = 0, 0 // CHECK-LABEL: _Z6fand_5b: -bool fand_5(bool a) { +bool fand_5(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:23 = M:0, C:2 return false && true; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:15 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:19 -> [[@LINE-1]]:23 = 0, 0 // CHECK-LABEL: _Z6fand_6b: -bool fand_6(bool a) { +bool fand_6(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:19 = M:0, C:2 return true && a; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:14 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:18 -> [[@LINE-1]]:19 = #2, (#1 - #2) // CHECK-LABEL: _Z6fand_7b: -bool fand_7(bool a) { +bool fand_7(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:20 = M:0, C:2 return a && false; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = #1, (#0 - #1) } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:20 = 0, 0 // CHECK-LABEL: _Z5for_0b: -bool for_0(bool a) { +bool for_0(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:19 = M:0, C:2 return true || a; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:14 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:18 -> [[@LINE-1]]:19 = (#1 - #2), #2 // CHECK-LABEL: _Z5for_1b: -bool for_1(bool a) { +bool for_1(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:20 = M:0, C:2 return a || false; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = (#0 - #1), #1 } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:20 = 0, 0 // CHECK-LABEL: _Z5for_2bb: -bool for_2(bool a, bool b) { +bool for_2(bool a, bool b) {// MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:24 = M:0, C:3 return true || a || b; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:14 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:18 -> [[@LINE-1]]:19 = (#3 - #4), #4 // CHECK: Branch,File 0, [[@LINE-2]]:23 -> [[@LINE-2]]:24 = (#1 - #2), #2 // CHECK-LABEL: _Z5for_3bb: -bool for_3(bool a, bool b) { +bool for_3(bool a, bool b) {// MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:25 = M:0, C:3 return a || false || b; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = (#0 - #3), #3 } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:20 = 0, 0 // CHECK: Branch,File 0, [[@LINE-2]]:24 -> [[@LINE-2]]:25 = (#1 - #2), #2 // CHECK-LABEL: _Z5for_4bb: -bool for_4(bool a, bool b) { +bool for_4(bool a, bool b) {// MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:24 = M:0, C:3 return a || b || true; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = (#0 - #3), #3 } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:16 = (#3 - #4), #4 // CHECK: Branch,File 0, [[@LINE-2]]:20 -> [[@LINE-2]]:24 = 0, 0 // CHECK-LABEL: _Z5for_5b: -bool for_5(bool a) { +bool for_5(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:23 = M:0, C:2 return true || false; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:14 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:18 -> [[@LINE-1]]:23 = 0, 0 // CHECK-LABEL: _Z5for_6b: -bool for_6(bool a) { +bool for_6(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:20 = M:0, C:2 return false || a; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:15 = 0, 0 } // CHECK: Branch,File 0, [[@LINE-1]]:19 -> [[@LINE-1]]:20 = (#1 - #2), #2 // CHECK-LABEL: _Z5for_7b: -bool for_7(bool a) { +bool for_7(bool a) { // MCDC: Decision,File 0, [[@LINE+1]]:10 -> [[@LINE+1]]:19 = M:0, C:2 return a || true; // CHECK: Branch,File 0, [[@LINE]]:10 -> [[@LINE]]:11 = (#0 - #1), #1 } // CHECK: Branch,File 0, [[@LINE-1]]:15 -> [[@LINE-1]]:19 = 0, 0 +// CHECK-LABEL: _Z5for_8b: +bool for_8(bool a) { // MCDC: Decision,File 0, [[@LINE+3]]:17 -> [[@LINE+3]]:30 = M:0, C:2 + // CHECK: Branch,File 0, [[@LINE+2]]:17 -> [[@LINE+2]]:21 = 0, 0 + // CHECK: Branch,File 0, [[@LINE+1]]:25 -> [[@LINE+1]]:30 = 0, 0 + if constexpr (true && false) + return true; + else + return false; +} diff --git a/clang/test/CoverageMapping/branch-mincounters.cpp b/clang/test/CoverageMapping/branch-mincounters.cpp index 6d4341cbeafd..aa1d3928aae2 100644 --- a/clang/test/CoverageMapping/branch-mincounters.cpp +++ b/clang/test/CoverageMapping/branch-mincounters.cpp @@ -2,6 +2,7 @@ // logical operators on branch conditions for branch coverage. // RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name branch-logical-mixed.cpp %s | FileCheck %s +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name branch-logical-mixed.cpp %s | FileCheck %s // CHECK-LABEL: _Z5func1ii: diff --git a/clang/test/CoverageMapping/branch-templates.cpp b/clang/test/CoverageMapping/branch-templates.cpp index 1fd01218c903..4e43626b8192 100644 --- a/clang/test/CoverageMapping/branch-templates.cpp +++ b/clang/test/CoverageMapping/branch-templates.cpp @@ -2,6 +2,7 @@ // instantiations. // RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name branch-templates.cpp %s | FileCheck %s +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name branch-templates.cpp %s | FileCheck %s template void unused(T x) { diff --git a/clang/test/CoverageMapping/if.cpp b/clang/test/CoverageMapping/if.cpp index 4326add3b344..65e3d62df79d 100644 --- a/clang/test/CoverageMapping/if.cpp +++ b/clang/test/CoverageMapping/if.cpp @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -fms-extensions -mllvm -emptyline-comment-coverage=false -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++23 -triple %itanium_abi_triple -main-file-name if.cpp %s | FileCheck %s +// RUN: %clang_cc1 -fms-extensions -mllvm -emptyline-comment-coverage=false -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++2b -triple %itanium_abi_triple -main-file-name if.cpp %s | FileCheck %s int nop() { return 0; } struct S { diff --git a/clang/test/CoverageMapping/logical.cpp b/clang/test/CoverageMapping/logical.cpp index cd1a388ec4c2..7de59e142980 100644 --- a/clang/test/CoverageMapping/logical.cpp +++ b/clang/test/CoverageMapping/logical.cpp @@ -1,22 +1,24 @@ // RUN: %clang_cc1 -mllvm -emptyline-comment-coverage=false -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name logical.cpp %s | FileCheck %s +// RUN: %clang_cc1 -mllvm -emptyline-comment-coverage=false -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name logical.cpp %s | FileCheck %s -check-prefix=MCDC -int main() { // CHECK: File 0, [[@LINE]]:12 -> [[@LINE+22]]:2 = #0 +int main() { // CHECK: File 0, [[@LINE]]:12 -> [[@LINE+23]]:2 = #0 bool bt = true; - bool bf = false; + bool bf = false; // MCDC: Decision,File 0, [[@LINE+1]]:12 -> [[@LINE+1]]:20 = M:0, C:2 bool a = bt && bf; // CHECK-NEXT: File 0, [[@LINE]]:12 -> [[@LINE]]:14 = #0 // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:12 -> [[@LINE-1]]:14 = #1, (#0 - #1) // CHECK-NEXT: File 0, [[@LINE-2]]:18 -> [[@LINE-2]]:20 = #1 // CHECK-NEXT: Branch,File 0, [[@LINE-3]]:18 -> [[@LINE-3]]:20 = #2, (#1 - #2) - + // MCDC: Decision,File 0, [[@LINE+1]]:7 -> [[@LINE+2]]:9 = M:1, C:2 a = bt && // CHECK-NEXT: File 0, [[@LINE]]:7 -> [[@LINE]]:9 = #0 bf; // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = #3, (#0 - #3) // CHECK-NEXT: File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = #3 // CHECK-NEXT: Branch,File 0, [[@LINE-2]]:7 -> [[@LINE-2]]:9 = #4, (#3 - #4) + // MCDC: Decision,File 0, [[@LINE+1]]:7 -> [[@LINE+1]]:15 = M:2, C:2 a = bf || bt; // CHECK-NEXT: File 0, [[@LINE]]:7 -> [[@LINE]]:9 = #0 // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = (#0 - #5), #5 // CHECK-NEXT: File 0, [[@LINE-2]]:13 -> [[@LINE-2]]:15 = #5 // CHECK-NEXT: Branch,File 0, [[@LINE-3]]:13 -> [[@LINE-3]]:15 = (#5 - #6), #6 - + // MCDC: Decision,File 0, [[@LINE+1]]:7 -> [[@LINE+2]]:9 = M:3, C:2 a = bf || // CHECK-NEXT: File 0, [[@LINE]]:7 -> [[@LINE]]:9 = #0 bt; // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = (#0 - #7), #7 // CHECK-NEXT: File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = #7 diff --git a/clang/test/CoverageMapping/mcdc-class.cpp b/clang/test/CoverageMapping/mcdc-class.cpp new file mode 100644 index 000000000000..dcf6123ee0fc --- /dev/null +++ b/clang/test/CoverageMapping/mcdc-class.cpp @@ -0,0 +1,31 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s | FileCheck %s + +extern void foo(); +extern void bar(); +class Value { + public: + void setValue( int len ); + int getValue( void ); + Value(); // This is the constructor declaration + ~Value(); // This is the destructor declaration + + private: + int value; +}; + +// Member functions definitions including constructor +Value::Value(void) { + if (value == 2 || value == 6) + foo(); +} +Value::~Value(void) { + if (value == 2 || value == 3) + bar(); +} + +// CHECK-LABEL: Decision,File 0, 18:7 -> 18:31 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 18:7 -> 18:17 = (#0 - #2), #2 [1,0,2] +// CHECK: Branch,File 0, 18:21 -> 18:31 = (#2 - #3), #3 [2,0,0] +// CHECK-LABEL: Decision,File 0, 22:7 -> 22:31 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 22:7 -> 22:17 = (#0 - #2), #2 [1,0,2] +// CHECK: Branch,File 0, 22:21 -> 22:31 = (#2 - #3), #3 [2,0,0] diff --git a/clang/test/CoverageMapping/mcdc-error-conditions.cpp b/clang/test/CoverageMapping/mcdc-error-conditions.cpp new file mode 100644 index 000000000000..d34ed6934347 --- /dev/null +++ b/clang/test/CoverageMapping/mcdc-error-conditions.cpp @@ -0,0 +1,7 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s 2>&1| FileCheck %s + +bool func_conditions(bool a, bool b, bool c, bool d, bool e, bool f, bool g) { + return a && b && c && d && e && f && g; +} + +// CHECK: warning: unsupported MC/DC boolean expression; number of conditions{{.*}} exceeds max diff --git a/clang/test/CoverageMapping/mcdc-error-nests.cpp b/clang/test/CoverageMapping/mcdc-error-nests.cpp new file mode 100644 index 000000000000..3add2b9ccd3f --- /dev/null +++ b/clang/test/CoverageMapping/mcdc-error-nests.cpp @@ -0,0 +1,10 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s 2>&1| FileCheck %s + +// "Split-nest" -- boolean expressions within boolean expressions. +extern bool bar(bool); +bool func_split_nest(bool a, bool b, bool c, bool d, bool e, bool f, bool g) { + bool res = a && b && c && bar(d && e) && f && g; + return bar(res); +} + +// CHECK: warning: unsupported MC/DC boolean expression; contains an operation with a nested boolean expression. diff --git a/clang/test/CoverageMapping/mcdc-logical-scalar-ids.cpp b/clang/test/CoverageMapping/mcdc-logical-scalar-ids.cpp new file mode 100644 index 000000000000..c820b5df5ad3 --- /dev/null +++ b/clang/test/CoverageMapping/mcdc-logical-scalar-ids.cpp @@ -0,0 +1,110 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s | FileCheck %s + +extern bool bar (bool, bool, bool, bool, bool); +bool func_scalar_and(bool a, bool b, bool c, bool d, bool e, bool f) { + bool res1 = a && b; + bool res2 = a && b && c; + bool res3 = a && b && c && d; + bool res4 = a && b && c && d && e; + bool res5 = a && b && c && d && e && f; + return bar(res1, res2, res3, res4, res5); +} + +// CHECK-LABEL: Decision,File 0, 5:17 -> 5:23 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 5:17 -> 5:18 = #1, (#0 - #1) [1,2,0] +// CHECK: Branch,File 0, 5:22 -> 5:23 = #2, (#1 - #2) [2,0,0] +// CHECK-LABEL: Decision,File 0, 6:17 -> 6:28 = M:1, C:3 +// CHECK-NEXT: Branch,File 0, 6:17 -> 6:18 = #5, (#0 - #5) [1,3,0] +// CHECK: Branch,File 0, 6:22 -> 6:23 = #6, (#5 - #6) [3,2,0] +// CHECK: Branch,File 0, 6:27 -> 6:28 = #4, (#3 - #4) [2,0,0] +// CHECK-LABEL: Decision,File 0, 7:17 -> 7:33 = M:2, C:4 +// CHECK-NEXT: Branch,File 0, 7:17 -> 7:18 = #11, (#0 - #11) [1,4,0] +// CHECK: Branch,File 0, 7:22 -> 7:23 = #12, (#11 - #12) [4,3,0] +// CHECK: Branch,File 0, 7:27 -> 7:28 = #10, (#9 - #10) [3,2,0] +// CHECK: Branch,File 0, 7:32 -> 7:33 = #8, (#7 - #8) [2,0,0] +// CHECK-LABEL: Decision,File 0, 8:17 -> 8:38 = M:4, C:5 +// CHECK-NEXT: Branch,File 0, 8:17 -> 8:18 = #19, (#0 - #19) [1,5,0] +// CHECK: Branch,File 0, 8:22 -> 8:23 = #20, (#19 - #20) [5,4,0] +// CHECK: Branch,File 0, 8:27 -> 8:28 = #18, (#17 - #18) [4,3,0] +// CHECK: Branch,File 0, 8:32 -> 8:33 = #16, (#15 - #16) [3,2,0] +// CHECK: Branch,File 0, 8:37 -> 8:38 = #14, (#13 - #14) [2,0,0] +// CHECK-LABEL: Decision,File 0, 9:17 -> 9:43 = M:8, C:6 +// CHECK-NEXT: Branch,File 0, 9:17 -> 9:18 = #29, (#0 - #29) [1,6,0] +// CHECK: Branch,File 0, 9:22 -> 9:23 = #30, (#29 - #30) [6,5,0] +// CHECK: Branch,File 0, 9:27 -> 9:28 = #28, (#27 - #28) [5,4,0] +// CHECK: Branch,File 0, 9:32 -> 9:33 = #26, (#25 - #26) [4,3,0] +// CHECK: Branch,File 0, 9:37 -> 9:38 = #24, (#23 - #24) [3,2,0] +// CHECK: Branch,File 0, 9:42 -> 9:43 = #22, (#21 - #22) [2,0,0] + +bool func_scalar_or(bool a, bool b, bool c, bool d, bool e, bool f) { + bool res1 = a || b; + bool res2 = a || b || c; + bool res3 = a || b || c || d; + bool res4 = a || b || c || d || e; + bool res5 = a || b || c || d || e || f; + return bar(res1, res2, res3, res4, res5); +} + +// CHECK-LABEL: Decision,File 0, 40:17 -> 40:23 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 40:17 -> 40:18 = (#0 - #1), #1 [1,0,2] +// CHECK: Branch,File 0, 40:22 -> 40:23 = (#1 - #2), #2 [2,0,0] +// CHECK-LABEL: Decision,File 0, 41:17 -> 41:28 = M:1, C:3 +// CHECK-NEXT: Branch,File 0, 41:17 -> 41:18 = (#0 - #5), #5 [1,0,3] +// CHECK: Branch,File 0, 41:22 -> 41:23 = (#5 - #6), #6 [3,0,2] +// CHECK: Branch,File 0, 41:27 -> 41:28 = (#3 - #4), #4 [2,0,0] +// CHECK-LABEL: Decision,File 0, 42:17 -> 42:33 = M:2, C:4 +// CHECK-NEXT: Branch,File 0, 42:17 -> 42:18 = (#0 - #11), #11 [1,0,4] +// CHECK: Branch,File 0, 42:22 -> 42:23 = (#11 - #12), #12 [4,0,3] +// CHECK: Branch,File 0, 42:27 -> 42:28 = (#9 - #10), #10 [3,0,2] +// CHECK: Branch,File 0, 42:32 -> 42:33 = (#7 - #8), #8 [2,0,0] +// CHECK-LABEL: Decision,File 0, 43:17 -> 43:38 = M:4, C:5 +// CHECK-NEXT: Branch,File 0, 43:17 -> 43:18 = (#0 - #19), #19 [1,0,5] +// CHECK: Branch,File 0, 43:22 -> 43:23 = (#19 - #20), #20 [5,0,4] +// CHECK: Branch,File 0, 43:27 -> 43:28 = (#17 - #18), #18 [4,0,3] +// CHECK: Branch,File 0, 43:32 -> 43:33 = (#15 - #16), #16 [3,0,2] +// CHECK: Branch,File 0, 43:37 -> 43:38 = (#13 - #14), #14 [2,0,0] +// CHECK-LABEL: Decision,File 0, 44:17 -> 44:43 = M:8, C:6 +// CHECK-NEXT: Branch,File 0, 44:17 -> 44:18 = (#0 - #29), #29 [1,0,6] +// CHECK: Branch,File 0, 44:22 -> 44:23 = (#29 - #30), #30 [6,0,5] +// CHECK: Branch,File 0, 44:27 -> 44:28 = (#27 - #28), #28 [5,0,4] +// CHECK: Branch,File 0, 44:32 -> 44:33 = (#25 - #26), #26 [4,0,3] +// CHECK: Branch,File 0, 44:37 -> 44:38 = (#23 - #24), #24 [3,0,2] +// CHECK: Branch,File 0, 44:42 -> 44:43 = (#21 - #22), #22 [2,0,0] + + +bool func_scalar_mix(bool a, bool b, bool c, bool d, bool e, bool f) { + bool res1 = a || b; + bool res2 = a && (b || c); + bool res3 = (a || b) && (c || d); + bool res4 = a && (b || c) && (d || e); + bool res5 = (a || b) && (c || d) && (e || f); + return bar(res1, res2, res3, res4, res5); +} + +// CHECK-LABEL: Decision,File 0, 76:17 -> 76:23 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 76:17 -> 76:18 = (#0 - #1), #1 [1,0,2] +// CHECK: Branch,File 0, 76:22 -> 76:23 = (#1 - #2), #2 [2,0,0] +// CHECK-LABEL: Decision,File 0, 77:17 -> 77:30 = M:1, C:3 +// CHECK-NEXT: Branch,File 0, 77:17 -> 77:18 = #3, (#0 - #3) [1,2,0] +// CHECK: Branch,File 0, 77:23 -> 77:24 = (#3 - #4), #4 [2,0,3] +// CHECK: Branch,File 0, 77:28 -> 77:29 = (#4 - #5), #5 [3,0,0] +// CHECK-LABEL: Decision,File 0, 78:17 -> 78:37 = M:2, C:4 +// CHECK-NEXT: File 0 +// CHECK-NEXT: Branch,File 0, 78:18 -> 78:19 = (#0 - #7), #7 [1,2,3] +// CHECK: Branch,File 0, 78:23 -> 78:24 = (#7 - #8), #8 [3,2,0] +// CHECK: Branch,File 0, 78:30 -> 78:31 = (#6 - #9), #9 [2,0,4] +// CHECK: Branch,File 0, 78:35 -> 78:36 = (#9 - #10), #10 [4,0,0] +// CHECK-LABEL: Decision,File 0, 79:17 -> 79:42 = M:4, C:5 +// CHECK-NEXT: Branch,File 0, 79:17 -> 79:18 = #12, (#0 - #12) [1,3,0] +// CHECK: Branch,File 0, 79:23 -> 79:24 = (#12 - #13), #13 [3,2,4] +// CHECK: Branch,File 0, 79:28 -> 79:29 = (#13 - #14), #14 [4,2,0] +// CHECK: Branch,File 0, 79:35 -> 79:36 = (#11 - #15), #15 [2,0,5] +// CHECK: Branch,File 0, 79:40 -> 79:41 = (#15 - #16), #16 [5,0,0] +// CHECK-LABEL: Decision,File 0, 80:17 -> 80:49 = M:8, C:6 +// CHECK-NEXT: File 0 +// CHECK-NEXT: Branch,File 0, 80:18 -> 80:19 = (#0 - #19), #19 [1,3,4] +// CHECK: Branch,File 0, 80:23 -> 80:24 = (#19 - #20), #20 [4,3,0] +// CHECK: Branch,File 0, 80:30 -> 80:31 = (#18 - #21), #21 [3,2,5] +// CHECK: Branch,File 0, 80:35 -> 80:36 = (#21 - #22), #22 [5,2,0] +// CHECK: Branch,File 0, 80:42 -> 80:43 = (#17 - #23), #23 [2,0,6] +// CHECK: Branch,File 0, 80:47 -> 80:48 = (#23 - #24), #24 [6,0,0] diff --git a/clang/test/CoverageMapping/mcdc-logical-stmt-ids-all.cpp b/clang/test/CoverageMapping/mcdc-logical-stmt-ids-all.cpp new file mode 100644 index 000000000000..5b13cc3507e9 --- /dev/null +++ b/clang/test/CoverageMapping/mcdc-logical-stmt-ids-all.cpp @@ -0,0 +1,131 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s | FileCheck %s + +bool func_if_and(bool a, bool b, bool c, bool d, bool e, bool f) { + if (a && b && c && d && e && f) + return true; + return false; +} + +// CHECK-LABEL: Decision,File 0, 4:7 -> 4:33 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 4:7 -> 4:8 = #10, (#0 - #10) [1,6,0] +// CHECK: Branch,File 0, 4:12 -> 4:13 = #11, (#10 - #11) [6,5,0] +// CHECK: Branch,File 0, 4:17 -> 4:18 = #9, (#8 - #9) [5,4,0] +// CHECK: Branch,File 0, 4:22 -> 4:23 = #7, (#6 - #7) [4,3,0] +// CHECK: Branch,File 0, 4:27 -> 4:28 = #5, (#4 - #5) [3,2,0] +// CHECK: Branch,File 0, 4:32 -> 4:33 = #3, (#2 - #3) [2,0,0] + +bool func_if_or(bool a, bool b, bool c, bool d, bool e, bool f) { + if (a || b || c || d || e || f) + return true; + return false; +} + +// CHECK-LABEL: Decision,File 0, 18:7 -> 18:33 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 18:7 -> 18:8 = (#0 - #10), #10 [1,0,6] +// CHECK: Branch,File 0, 18:12 -> 18:13 = (#10 - #11), #11 [6,0,5] +// CHECK: Branch,File 0, 18:17 -> 18:18 = (#8 - #9), #9 [5,0,4] +// CHECK: Branch,File 0, 18:22 -> 18:23 = (#6 - #7), #7 [4,0,3] +// CHECK: Branch,File 0, 18:27 -> 18:28 = (#4 - #5), #5 [3,0,2] +// CHECK: Branch,File 0, 18:32 -> 18:33 = (#2 - #3), #3 [2,0,0] + +bool func_while_and(bool a, bool b, bool c, bool d, bool e, bool f) { + while (a && b && c && d && e && f) { return true; } + return false; +} + +// CHECK-LABEL: Decision,File 0, 32:10 -> 32:36 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 32:10 -> 32:11 = #10, (#0 - #10) [1,6,0] +// CHECK: Branch,File 0, 32:15 -> 32:16 = #11, (#10 - #11) [6,5,0] +// CHECK: Branch,File 0, 32:20 -> 32:21 = #9, (#8 - #9) [5,4,0] +// CHECK: Branch,File 0, 32:25 -> 32:26 = #7, (#6 - #7) [4,3,0] +// CHECK: Branch,File 0, 32:30 -> 32:31 = #5, (#4 - #5) [3,2,0] +// CHECK: Branch,File 0, 32:35 -> 32:36 = #3, (#2 - #3) [2,0,0] + +bool func_while_or(bool a, bool b, bool c, bool d, bool e, bool f) { + while (a || b || c || d || e || f) { return true; } + return false; +} + +// CHECK-LABEL: Decision,File 0, 45:10 -> 45:36 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 45:10 -> 45:11 = (#0 - #10), #10 [1,0,6] +// CHECK: Branch,File 0, 45:15 -> 45:16 = (#10 - #11), #11 [6,0,5] +// CHECK: Branch,File 0, 45:20 -> 45:21 = (#8 - #9), #9 [5,0,4] +// CHECK: Branch,File 0, 45:25 -> 45:26 = (#6 - #7), #7 [4,0,3] +// CHECK: Branch,File 0, 45:30 -> 45:31 = (#4 - #5), #5 [3,0,2] +// CHECK: Branch,File 0, 45:35 -> 45:36 = (#2 - #3), #3 [2,0,0] + +bool func_for_and(bool a, bool b, bool c, bool d, bool e, bool f) { + for (;a && b && c && d && e && f;) { return true; } + return false; +} + +// CHECK-LABEL: Decision,File 0, 58:9 -> 58:35 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 58:9 -> 58:10 = #10, (#0 - #10) [1,6,0] +// CHECK: Branch,File 0, 58:14 -> 58:15 = #11, (#10 - #11) [6,5,0] +// CHECK: Branch,File 0, 58:19 -> 58:20 = #9, (#8 - #9) [5,4,0] +// CHECK: Branch,File 0, 58:24 -> 58:25 = #7, (#6 - #7) [4,3,0] +// CHECK: Branch,File 0, 58:29 -> 58:30 = #5, (#4 - #5) [3,2,0] +// CHECK: Branch,File 0, 58:34 -> 58:35 = #3, (#2 - #3) [2,0,0] + +bool func_for_or(bool a, bool b, bool c, bool d, bool e, bool f) { + for (;a || b || c || d || e || f;) { return true; } + return false; +} + +// CHECK-LABEL: Decision,File 0, 71:9 -> 71:35 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 71:9 -> 71:10 = (#0 - #10), #10 [1,0,6] +// CHECK: Branch,File 0, 71:14 -> 71:15 = (#10 - #11), #11 [6,0,5] +// CHECK: Branch,File 0, 71:19 -> 71:20 = (#8 - #9), #9 [5,0,4] +// CHECK: Branch,File 0, 71:24 -> 71:25 = (#6 - #7), #7 [4,0,3] +// CHECK: Branch,File 0, 71:29 -> 71:30 = (#4 - #5), #5 [3,0,2] +// CHECK: Branch,File 0, 71:34 -> 71:35 = (#2 - #3), #3 [2,0,0] + +bool func_do_and(bool a, bool b, bool c, bool d, bool e, bool f) { + do {} while (a && b && c && d && e && f); + return false; +} + +// CHECK-LABEL: Decision,File 0, 84:16 -> 84:42 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 84:16 -> 84:17 = #10, ((#0 + #1) - #10) [1,6,0] +// CHECK: Branch,File 0, 84:21 -> 84:22 = #11, (#10 - #11) [6,5,0] +// CHECK: Branch,File 0, 84:26 -> 84:27 = #9, (#8 - #9) [5,4,0] +// CHECK: Branch,File 0, 84:31 -> 84:32 = #7, (#6 - #7) [4,3,0] +// CHECK: Branch,File 0, 84:36 -> 84:37 = #5, (#4 - #5) [3,2,0] +// CHECK: Branch,File 0, 84:41 -> 84:42 = #3, (#2 - #3) [2,0,0] + +bool func_do_or(bool a, bool b, bool c, bool d, bool e, bool f) { + do {} while (a || b || c || d || e || f); + return false; +} + +// CHECK-LABEL: Decision,File 0, 97:16 -> 97:42 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 97:16 -> 97:17 = ((#0 + #1) - #10), #10 [1,0,6] +// CHECK: Branch,File 0, 97:21 -> 97:22 = (#10 - #11), #11 [6,0,5] +// CHECK: Branch,File 0, 97:26 -> 97:27 = (#8 - #9), #9 [5,0,4] +// CHECK: Branch,File 0, 97:31 -> 97:32 = (#6 - #7), #7 [4,0,3] +// CHECK: Branch,File 0, 97:36 -> 97:37 = (#4 - #5), #5 [3,0,2] +// CHECK: Branch,File 0, 97:41 -> 97:42 = (#2 - #3), #3 [2,0,0] + +bool func_ternary_and(bool a, bool b, bool c, bool d, bool e, bool f) { + return (a && b && c && d && e && f) ? true : false; +} + +// CHECK-LABEL: Decision,File 0, 110:11 -> 110:37 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 110:11 -> 110:12 = #10, (#0 - #10) [1,6,0] +// CHECK: Branch,File 0, 110:16 -> 110:17 = #11, (#10 - #11) [6,5,0] +// CHECK: Branch,File 0, 110:21 -> 110:22 = #9, (#8 - #9) [5,4,0] +// CHECK: Branch,File 0, 110:26 -> 110:27 = #7, (#6 - #7) [4,3,0] +// CHECK: Branch,File 0, 110:31 -> 110:32 = #5, (#4 - #5) [3,2,0] +// CHECK: Branch,File 0, 110:36 -> 110:37 = #3, (#2 - #3) [2,0,0] + +bool func_ternary_or(bool a, bool b, bool c, bool d, bool e, bool f) { + return (a || b || c || d || e || f) ? true : false; +} + +// CHECK-LABEL: Decision,File 0, 122:11 -> 122:37 = M:0, C:6 +// CHECK-NEXT: Branch,File 0, 122:11 -> 122:12 = (#0 - #10), #10 [1,0,6] +// CHECK: Branch,File 0, 122:16 -> 122:17 = (#10 - #11), #11 [6,0,5] +// CHECK: Branch,File 0, 122:21 -> 122:22 = (#8 - #9), #9 [5,0,4] +// CHECK: Branch,File 0, 122:26 -> 122:27 = (#6 - #7), #7 [4,0,3] +// CHECK: Branch,File 0, 122:31 -> 122:32 = (#4 - #5), #5 [3,0,2] +// CHECK: Branch,File 0, 122:36 -> 122:37 = (#2 - #3), #3 [2,0,0] diff --git a/clang/test/CoverageMapping/mcdc-logical-stmt-ids.cpp b/clang/test/CoverageMapping/mcdc-logical-stmt-ids.cpp new file mode 100644 index 000000000000..99854ec27a3f --- /dev/null +++ b/clang/test/CoverageMapping/mcdc-logical-stmt-ids.cpp @@ -0,0 +1,111 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only %s | FileCheck %s + +bool func_if_and(bool a, bool b, bool c, bool d, bool e, bool f) { + if (a && b) + if (a && b && c) + if (a && b && c && d) + if (a && b && c && d && e) + if (a && b && c && d && e && f) + return true; + return false; +} + +// CHECK-LABEL: Decision,File 0, 4:7 -> 4:13 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 4:7 -> 4:8 = #2, (#0 - #2) [1,2,0] +// CHECK: Branch,File 0, 4:12 -> 4:13 = #3, (#2 - #3) [2,0,0] +// CHECK-LABEL: Decision,File 0, 5:9 -> 5:20 = M:1, C:3 +// CHECK-NEXT: Branch,File 0, 5:9 -> 5:10 = #7, (#1 - #7) [1,3,0] +// CHECK: Branch,File 0, 5:14 -> 5:15 = #8, (#7 - #8) [3,2,0] +// CHECK: Branch,File 0, 5:19 -> 5:20 = #6, (#5 - #6) [2,0,0] +// CHECK-LABEL: Decision,File 0, 6:11 -> 6:27 = M:2, C:4 +// CHECK-NEXT: Branch,File 0, 6:11 -> 6:12 = #14, (#4 - #14) [1,4,0] +// CHECK: Branch,File 0, 6:16 -> 6:17 = #15, (#14 - #15) [4,3,0] +// CHECK: Branch,File 0, 6:21 -> 6:22 = #13, (#12 - #13) [3,2,0] +// CHECK: Branch,File 0, 6:26 -> 6:27 = #11, (#10 - #11) [2,0,0] +// CHECK-LABEL: Decision,File 0, 7:13 -> 7:34 = M:4, C:5 +// CHECK-NEXT: Branch,File 0, 7:13 -> 7:14 = #23, (#9 - #23) [1,5,0] +// CHECK: Branch,File 0, 7:18 -> 7:19 = #24, (#23 - #24) [5,4,0] +// CHECK: Branch,File 0, 7:23 -> 7:24 = #22, (#21 - #22) [4,3,0] +// CHECK: Branch,File 0, 7:28 -> 7:29 = #20, (#19 - #20) [3,2,0] +// CHECK: Branch,File 0, 7:33 -> 7:34 = #18, (#17 - #18) [2,0,0] +// CHECK-LABEL: Decision,File 0, 8:16 -> 8:42 = M:8, C:6 +// CHECK-NEXT: Branch,File 0, 8:16 -> 8:17 = #34, (#16 - #34) [1,6,0] +// CHECK: Branch,File 0, 8:21 -> 8:22 = #35, (#34 - #35) [6,5,0] +// CHECK: Branch,File 0, 8:26 -> 8:27 = #33, (#32 - #33) [5,4,0] +// CHECK: Branch,File 0, 8:31 -> 8:32 = #31, (#30 - #31) [4,3,0] +// CHECK: Branch,File 0, 8:36 -> 8:37 = #29, (#28 - #29) [3,2,0] +// CHECK: Branch,File 0, 8:41 -> 8:42 = #27, (#26 - #27) [2,0,0] + +bool func_if_or(bool a, bool b, bool c, bool d, bool e, bool f) { + if (a || b) + if (a || b || c) + if (a || b || c || d) + if (a || b || c || d || e) + if (a || b || c || d || e || f) + return true; + return false; +} + +// CHECK-LABEL: Decision,File 0, 40:7 -> 40:13 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 40:7 -> 40:8 = (#0 - #2), #2 [1,0,2] +// CHECK: Branch,File 0, 40:12 -> 40:13 = (#2 - #3), #3 [2,0,0] +// CHECK-LABEL: Decision,File 0, 41:9 -> 41:20 = M:1, C:3 +// CHECK-NEXT: Branch,File 0, 41:9 -> 41:10 = (#1 - #7), #7 [1,0,3] +// CHECK: Branch,File 0, 41:14 -> 41:15 = (#7 - #8), #8 [3,0,2] +// CHECK: Branch,File 0, 41:19 -> 41:20 = (#5 - #6), #6 [2,0,0] +// CHECK-LABEL: Decision,File 0, 42:11 -> 42:27 = M:2, C:4 +// CHECK-NEXT: Branch,File 0, 42:11 -> 42:12 = (#4 - #14), #14 [1,0,4] +// CHECK: Branch,File 0, 42:16 -> 42:17 = (#14 - #15), #15 [4,0,3] +// CHECK: Branch,File 0, 42:21 -> 42:22 = (#12 - #13), #13 [3,0,2] +// CHECK: Branch,File 0, 42:26 -> 42:27 = (#10 - #11), #11 [2,0,0] +// CHECK-LABEL: Decision,File 0, 43:13 -> 43:34 = M:4, C:5 +// CHECK-NEXT: Branch,File 0, 43:13 -> 43:14 = (#9 - #23), #23 [1,0,5] +// CHECK: Branch,File 0, 43:18 -> 43:19 = (#23 - #24), #24 [5,0,4] +// CHECK: Branch,File 0, 43:23 -> 43:24 = (#21 - #22), #22 [4,0,3] +// CHECK: Branch,File 0, 43:28 -> 43:29 = (#19 - #20), #20 [3,0,2] +// CHECK: Branch,File 0, 43:33 -> 43:34 = (#17 - #18), #18 [2,0,0] +// CHECK-LABEL: Decision,File 0, 44:16 -> 44:42 = M:8, C:6 +// CHECK-NEXT: Branch,File 0, 44:16 -> 44:17 = (#16 - #34), #34 [1,0,6] +// CHECK: Branch,File 0, 44:21 -> 44:22 = (#34 - #35), #35 [6,0,5] +// CHECK: Branch,File 0, 44:26 -> 44:27 = (#32 - #33), #33 [5,0,4] +// CHECK: Branch,File 0, 44:31 -> 44:32 = (#30 - #31), #31 [4,0,3] +// CHECK: Branch,File 0, 44:36 -> 44:37 = (#28 - #29), #29 [3,0,2] +// CHECK: Branch,File 0, 44:41 -> 44:42 = (#26 - #27), #27 [2,0,0] + +bool func_if_mix(bool a, bool b, bool c, bool d, bool e, bool f) { + if (a || b) + if (a && (b || c)) + if ((a || b) && (c || d)) + if (a && (b || c) && (d || e)) + if ((a || b) && (c || d) && (e || f)) + return true; + return false; +} + +// CHECK-LABEL: Decision,File 0, 76:7 -> 76:13 = M:0, C:2 +// CHECK-NEXT: Branch,File 0, 76:7 -> 76:8 = (#0 - #2), #2 [1,0,2] +// CHECK: Branch,File 0, 76:12 -> 76:13 = (#2 - #3), #3 [2,0,0] +// CHECK-LABEL: Decision,File 0, 77:9 -> 77:22 = M:1, C:3 +// CHECK-NEXT: Branch,File 0, 77:9 -> 77:10 = #5, (#1 - #5) [1,2,0] +// CHECK: Branch,File 0, 77:15 -> 77:16 = (#5 - #6), #6 [2,0,3] +// CHECK: Branch,File 0, 77:20 -> 77:21 = (#6 - #7), #7 [3,0,0] +// CHECK-LABEL: Decision,File 0, 78:11 -> 78:31 = M:2, C:4 +// CHECK-NEXT: File 0 +// CHECK-NEXT: Branch,File 0, 78:12 -> 78:13 = (#4 - #10), #10 [1,2,3] +// CHECK: Branch,File 0, 78:17 -> 78:18 = (#10 - #11), #11 [3,2,0] +// CHECK: Branch,File 0, 78:24 -> 78:25 = (#9 - #12), #12 [2,0,4] +// CHECK: Branch,File 0, 78:29 -> 78:30 = (#12 - #13), #13 [4,0,0] +// CHECK-LABEL: Decision,File 0, 79:13 -> 79:38 = M:4, C:5 +// CHECK-NEXT: Branch,File 0, 79:13 -> 79:14 = #16, (#8 - #16) [1,3,0] +// CHECK: Branch,File 0, 79:19 -> 79:20 = (#16 - #17), #17 [3,2,4] +// CHECK: Branch,File 0, 79:24 -> 79:25 = (#17 - #18), #18 [4,2,0] +// CHECK: Branch,File 0, 79:31 -> 79:32 = (#15 - #19), #19 [2,0,5] +// CHECK: Branch,File 0, 79:36 -> 79:37 = (#19 - #20), #20 [5,0,0] +// CHECK-LABEL: Decision,File 0, 80:15 -> 80:47 = M:8, C:6 +// CHECK-NEXT: File 0 +// CHECK-NEXT: Branch,File 0, 80:16 -> 80:17 = (#14 - #24), #24 [1,3,4] +// CHECK: Branch,File 0, 80:21 -> 80:22 = (#24 - #25), #25 [4,3,0] +// CHECK: Branch,File 0, 80:28 -> 80:29 = (#23 - #26), #26 [3,2,5] +// CHECK: Branch,File 0, 80:33 -> 80:34 = (#26 - #27), #27 [5,2,0] +// CHECK: Branch,File 0, 80:40 -> 80:41 = (#22 - #28), #28 [2,0,6] +// CHECK: Branch,File 0, 80:45 -> 80:46 = (#28 - #29), #29 [6,0,0] diff --git a/clang/test/Profile/c-linkage-available_externally.c b/clang/test/Profile/c-linkage-available_externally.c index f6375e15f451..010f191dc5e1 100644 --- a/clang/test/Profile/c-linkage-available_externally.c +++ b/clang/test/Profile/c-linkage-available_externally.c @@ -1,8 +1,10 @@ // Make sure instrumentation data from available_externally functions doesn't // get thrown out and are emitted with the expected linkage. // RUN: %clang_cc1 -O2 -triple x86_64-apple-macosx10.9 -main-file-name c-linkage-available_externally.c %s -o - -emit-llvm -fprofile-instrument=clang | FileCheck %s +// RUN: %clang_cc1 -fcoverage-mcdc -O2 -triple x86_64-apple-macosx10.9 -main-file-name c-linkage-available_externally.c %s -o - -emit-llvm -fprofile-instrument=clang | FileCheck %s -check-prefix=MCDC // CHECK: @__profc_foo = linkonce_odr hidden global [1 x i64] zeroinitializer, section "__DATA,__llvm_prf_cnts", align 8 +// MCDC: @__profbm_foo = linkonce_odr hidden global [0 x i8] zeroinitializer, section "__DATA,__llvm_prf_bits", align 1 // CHECK: @__profd_foo = linkonce_odr hidden global {{.*}} i64 sub (i64 ptrtoint (ptr @__profc_foo to i64), i64 ptrtoint (ptr @__profd_foo to i64)), {{.*}}, section "__DATA,__llvm_prf_data,regular,live_support", align 8 inline int foo(void) { return 1; } diff --git a/clang/test/Profile/c-mcdc-class.cpp b/clang/test/Profile/c-mcdc-class.cpp new file mode 100644 index 000000000000..2206a39ee4ff --- /dev/null +++ b/clang/test/Profile/c-mcdc-class.cpp @@ -0,0 +1,104 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -fcoverage-mcdc | FileCheck %s -check-prefix=MCDCCTOR +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -fcoverage-mcdc | FileCheck %s -check-prefix=MCDCDTOR + +extern void foo(); +extern void bar(); +class Value { + public: + void setValue(int len); + int getValue(void); + Value(); // This is the constructor declaration + ~Value(); // This is the destructor declaration + + private: + int value; +}; + +// Member functions definitions including constructor +Value::Value(void) { + if (value != 2 || value != 6) + foo(); +} +Value::~Value(void) { + if (value != 2 || value != 3) + bar(); +} + +// MCDC BOOKKEEPING. +// MCDCCTOR: @__profbm__ZN5ValueC2Ev = private global [1 x i8] zeroinitializer +// MCDCCTOR: @__profc__ZN5ValueC2Ev = private global [4 x i64] zeroinitializer + +// ALLOCATE MCDC TEMP AND ZERO IT. +// MCDCCTOR-LABEL: @_ZN5ValueC2Ev( +// MCDCCTOR: %mcdc.addr = alloca i32, align 4 +// MCDCCTOR: store i32 0, ptr %mcdc.addr, align 4 + +// SHIFT FIRST CONDITION WITH ID = 0. +// MCDCCTOR: %[[LAB1:[0-9]+]] = load i32, ptr %value, align 4 +// MCDCCTOR-DAG: %[[BOOL:cmp[0-9]*]] = icmp ne i32 %[[LAB1]], 2 +// MCDCCTOR-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDCCTOR-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDCCTOR-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 0 +// MCDCCTOR-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDCCTOR-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT SECOND CONDITION WITH ID = 1. +// MCDCCTOR: %[[LAB1:[0-9]+]] = load i32, ptr %value2, align 4 +// MCDCCTOR-DAG: %[[BOOL:cmp[0-9]*]] = icmp ne i32 %[[LAB1]], 6 +// MCDCCTOR-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDCCTOR-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDCCTOR-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 1 +// MCDCCTOR-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDCCTOR-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// UPDATE FINAL BITMASK WITH RESULT. +// MCDCCTOR-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDCCTOR: %[[LAB1:[0-9]+]] = lshr i32 %[[TEMP]], 3 +// MCDCCTOR: %[[LAB2:[0-9]+]] = zext i32 %[[LAB1]] to i64 +// MCDCCTOR: %[[LAB3:[0-9]+]] = add i64 ptrtoint (ptr @__profbm__ZN5ValueC2Ev to i64), %[[LAB2]] +// MCDCCTOR: %[[LAB4:[0-9]+]] = inttoptr i64 %[[LAB3]] to ptr +// MCDCCTOR: %[[LAB5:[0-9]+]] = and i32 %[[TEMP]], 7 +// MCDCCTOR: %[[LAB6:[0-9]+]] = trunc i32 %[[LAB5]] to i8 +// MCDCCTOR: %[[LAB7:[0-9]+]] = shl i8 1, %[[LAB6]] +// MCDCCTOR: %mcdc.bits = load i8, ptr %[[LAB4]], align 1 +// MCDCCTOR: %[[LAB8:[0-9]+]] = or i8 %mcdc.bits, %[[LAB7]] +// MCDCCTOR: store i8 %[[LAB8]], ptr %[[LAB4]], align 1 + +// MCDCDTOR: @__profbm__ZN5ValueD2Ev = private global [1 x i8] zeroinitializer +// MCDCDTOR: @__profc__ZN5ValueD2Ev = private global [4 x i64] zeroinitializer + +// ALLOCATE MCDC TEMP AND ZERO IT. +// MCDCDTOR-LABEL: @_ZN5ValueD2Ev( +// MCDCDTOR: %mcdc.addr = alloca i32, align 4 +// MCDCDTOR: store i32 0, ptr %mcdc.addr, align 4 + +// SHIFT FIRST CONDITION WITH ID = 0. +// MCDCDTOR: %[[LAB1:[0-9]+]] = load i32, ptr %value, align 4 +// MCDCDTOR-DAG: %[[BOOL:cmp[0-9]*]] = icmp ne i32 %[[LAB1]], 2 +// MCDCDTOR-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDCDTOR-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDCDTOR-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 0 +// MCDCDTOR-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDCDTOR-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT SECOND CONDITION WITH ID = 1. +// MCDCDTOR: %[[LAB1:[0-9]+]] = load i32, ptr %value2, align 4 +// MCDCDTOR-DAG: %[[BOOL:cmp[0-9]*]] = icmp ne i32 %[[LAB1]], 3 +// MCDCDTOR-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDCDTOR-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDCDTOR-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 1 +// MCDCDTOR-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDCDTOR-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// UPDATE FINAL BITMASK WITH RESULT. +// MCDCDTOR-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDCDTOR: %[[LAB1:[0-9]+]] = lshr i32 %[[TEMP]], 3 +// MCDCDTOR: %[[LAB2:[0-9]+]] = zext i32 %[[LAB1]] to i64 +// MCDCDTOR: %[[LAB3:[0-9]+]] = add i64 ptrtoint (ptr @__profbm__ZN5ValueD2Ev to i64), %[[LAB2]] +// MCDCDTOR: %[[LAB4:[0-9]+]] = inttoptr i64 %[[LAB3]] to ptr +// MCDCDTOR: %[[LAB5:[0-9]+]] = and i32 %[[TEMP]], 7 +// MCDCDTOR: %[[LAB6:[0-9]+]] = trunc i32 %[[LAB5]] to i8 +// MCDCDTOR: %[[LAB7:[0-9]+]] = shl i8 1, %[[LAB6]] +// MCDCDTOR: %mcdc.bits = load i8, ptr %[[LAB4]], align 1 +// MCDCDTOR: %[[LAB8:[0-9]+]] = or i8 %mcdc.bits, %[[LAB7]] +// MCDCDTOR: store i8 %[[LAB8]], ptr %[[LAB4]], align 1 diff --git a/clang/test/Profile/c-mcdc-nested-ternary.c b/clang/test/Profile/c-mcdc-nested-ternary.c new file mode 100644 index 000000000000..4b014e07f6df --- /dev/null +++ b/clang/test/Profile/c-mcdc-nested-ternary.c @@ -0,0 +1,68 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -fcoverage-mcdc | FileCheck %s -check-prefix=MCDC +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping | FileCheck %s -check-prefix=NOMCDC + +int test(int b, int c, int d, int e, int f) { + return ((b ? c : d) && e && f); +} + +// NOMCDC-NOT: %mcdc.addr +// NOMCDC-NOT: __profbm_test + +// MCDC BOOKKEEPING. +// MCDC: @__profbm_test = private global [1 x i8] zeroinitializer + +// ALLOCATE MCDC TEMP AND ZERO IT. +// MCDC-LABEL: @test( +// MCDC: %mcdc.addr = alloca i32, align 4 +// MCDC: store i32 0, ptr %mcdc.addr, align 4 + +// TERNARY TRUE SHOULD SHIFT ID = 0 FOR CONDITION 'c'. +// MCDC-LABEL: cond.true: +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %c.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 0 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// TERNARY FALSE SHOULD SHIFT ID = 0 FOR CONDITION 'd'. +// MCDC-LABEL: cond.false: +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %d.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 0 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT SECOND CONDITION WITH ID = 2. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %e.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 2 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT THIRD CONDITION WITH ID = 1. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %f.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 1 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// UPDATE FINAL BITMASK WITH RESULT. +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC: %[[LAB1:[0-9]+]] = lshr i32 %[[TEMP]], 3 +// MCDC: %[[LAB2:[0-9]+]] = zext i32 %[[LAB1]] to i64 +// MCDC: %[[LAB3:[0-9]+]] = add i64 ptrtoint (ptr @__profbm_test to i64), %[[LAB2]] +// MCDC: %[[LAB4:[0-9]+]] = inttoptr i64 %[[LAB3]] to ptr +// MCDC: %[[LAB5:[0-9]+]] = and i32 %[[TEMP]], 7 +// MCDC: %[[LAB6:[0-9]+]] = trunc i32 %[[LAB5]] to i8 +// MCDC: %[[LAB7:[0-9]+]] = shl i8 1, %[[LAB6]] +// MCDC: %mcdc.bits = load i8, ptr %[[LAB4]], align 1 +// MCDC: %[[LAB8:[0-9]+]] = or i8 %mcdc.bits, %[[LAB7]] +// MCDC: store i8 %[[LAB8]], ptr %[[LAB4]], align 1 diff --git a/clang/test/Profile/c-mcdc-not.c b/clang/test/Profile/c-mcdc-not.c new file mode 100644 index 000000000000..aa638b9680b8 --- /dev/null +++ b/clang/test/Profile/c-mcdc-not.c @@ -0,0 +1,88 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -fcoverage-mcdc | FileCheck %s -check-prefix=MCDC +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping | FileCheck %s -check-prefix=NOMCDC + +int test(int a, int b, int c, int d, int e, int f) { + return ((!a && b) || ((!c && d) || (e && !f))); +} + +// NOMCDC-NOT: %mcdc.addr +// NOMCDC-NOT: __profbm_test + +// MCDC BOOKKEEPING. +// MCDC: @__profbm_test = private global [8 x i8] zeroinitializer +// MCDC: @__profc_test = private global [9 x i64] zeroinitializer + +// ALLOCATE MCDC TEMP AND ZERO IT. +// MCDC-LABEL: @test( +// MCDC: %mcdc.addr = alloca i32, align 4 +// MCDC: store i32 0, ptr %mcdc.addr, align 4 + +// SHIFT FIRST CONDITION WITH ID = 0. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %a.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[LNOT:lnot[0-9]*]] = xor i1 %[[BOOL]] +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[LNOT]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 0 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT SECOND CONDITION WITH ID = 2. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %b.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 2 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT THIRD CONDITION WITH ID = 1. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %c.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[LNOT:lnot[0-9]*]] = xor i1 %[[BOOL]] +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[LNOT]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 1 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT FOURTH CONDITION WITH ID = 4. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %d.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 4 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT FIFTH CONDITION WITH ID = 3. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %e.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 3 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT SIXTH CONDITION WITH ID = 5. +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %f.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[LNOT:lnot[0-9]*]] = xor i1 %[[BOOL]] +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[LNOT]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 5 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// UPDATE FINAL BITMASK WITH RESULT. +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC: %[[LAB1:[0-9]+]] = lshr i32 %[[TEMP]], 3 +// MCDC: %[[LAB2:[0-9]+]] = zext i32 %[[LAB1]] to i64 +// MCDC: %[[LAB3:[0-9]+]] = add i64 ptrtoint (ptr @__profbm_test to i64), %[[LAB2]] +// MCDC: %[[LAB4:[0-9]+]] = inttoptr i64 %[[LAB3]] to ptr +// MCDC: %[[LAB5:[0-9]+]] = and i32 %[[TEMP]], 7 +// MCDC: %[[LAB6:[0-9]+]] = trunc i32 %[[LAB5]] to i8 +// MCDC: %[[LAB7:[0-9]+]] = shl i8 1, %[[LAB6]] +// MCDC: %mcdc.bits = load i8, ptr %[[LAB4]], align 1 +// MCDC: %[[LAB8:[0-9]+]] = or i8 %mcdc.bits, %[[LAB7]] +// MCDC: store i8 %[[LAB8]], ptr %[[LAB4]], align 1 diff --git a/clang/test/Profile/c-mcdc.c b/clang/test/Profile/c-mcdc.c new file mode 100644 index 000000000000..ac845d204853 --- /dev/null +++ b/clang/test/Profile/c-mcdc.c @@ -0,0 +1,102 @@ +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -fcoverage-mcdc | FileCheck %s -check-prefix=MCDC +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping | FileCheck %s -check-prefix=NOMCDC +// RUN: %clang_cc1 -triple %itanium_abi_triple %s -o - -emit-llvm -fprofile-instrument=clang -fcoverage-mapping -fcoverage-mcdc -disable-llvm-passes | FileCheck %s -check-prefix=NOPROFPASS + +int test(int a, int b, int c, int d, int e, int f) { + return ((a && b) || ((c && d) || (e && f))); +} + +// NOMCDC-NOT: %mcdc.addr +// NOMCDC-NOT: __profbm_test +// NOPROFPASS-NOT: __profbm_test + +// MCDC BOOKKEEPING. +// MCDC: @__profbm_test = private global [8 x i8] zeroinitializer +// MCDC: @__profc_test = private global [9 x i64] zeroinitializer + +// ALLOCATE MCDC TEMP AND ZERO IT. +// NOPROFPASS-LABEL: @test( +// NOPROFPASS: call void @llvm.instrprof.mcdc.parameters(ptr @__profn_test, i64 [[HASH:[0-9]+]], i32 8) +// MCDC-LABEL: @test( +// MCDC: %mcdc.addr = alloca i32, align 4 +// MCDC: store i32 0, ptr %mcdc.addr, align 4 + +// SHIFT FIRST CONDITION WITH ID = 0. +// NOPROFPASS: call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 [[HASH]], i32 0, ptr %mcdc.addr, i1 %tobool{{[0-9]*}}) +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %a.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 0 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT SECOND CONDITION WITH ID = 2. +// NOPROFPASS-LABEL: land.lhs.true: +// NOPROFPASS: call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 [[HASH]], i32 2, ptr %mcdc.addr, i1 %tobool{{[0-9]*}}) +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %b.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 2 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT THIRD CONDITION WITH ID = 1. +// NOPROFPASS-LABEL: lor.rhs: +// NOPROFPASS: call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 [[HASH]], i32 1, ptr %mcdc.addr, i1 %tobool{{[0-9]*}}) +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %c.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 1 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT FOURTH CONDITION WITH ID = 4. +// NOPROFPASS-LABEL: land.lhs.true3: +// NOPROFPASS: call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 [[HASH]], i32 4, ptr %mcdc.addr, i1 %tobool{{[0-9]*}}) +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %d.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 4 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT FIFTH CONDITION WITH ID = 3. +// NOPROFPASS-LABEL: lor.rhs6: +// NOPROFPASS: call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 [[HASH]], i32 3, ptr %mcdc.addr, i1 %tobool{{[0-9]*}}) +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %e.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 3 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// SHIFT SIXTH CONDITION WITH ID = 5. +// NOPROFPASS-LABEL: land.rhs: +// NOPROFPASS: call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 [[HASH]], i32 5, ptr %mcdc.addr, i1 %tobool{{[0-9]*}}) +// MCDC: %[[LAB1:[0-9]+]] = load i32, ptr %f.addr, align 4 +// MCDC-DAG: %[[BOOL:tobool[0-9]*]] = icmp ne i32 %[[LAB1]], 0 +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC-DAG: %[[LAB2:[0-9]+]] = zext i1 %[[BOOL]] to i32 +// MCDC-DAG: %[[LAB3:[0-9]+]] = shl i32 %[[LAB2]], 5 +// MCDC-DAG: %[[LAB4:[0-9]+]] = or i32 %[[TEMP]], %[[LAB3]] +// MCDC-DAG: store i32 %[[LAB4]], ptr %mcdc.addr, align 4 + +// UPDATE FINAL BITMASK WITH RESULT. +// NOPROFPASS-LABEL: lor.end: +// NOPROFPASS: call void @llvm.instrprof.mcdc.tvbitmap.update(ptr @__profn_test, i64 [[HASH]], i32 8, i32 0, ptr %mcdc.addr) +// MCDC-DAG: %[[TEMP:mcdc.temp[0-9]*]] = load i32, ptr %mcdc.addr, align 4 +// MCDC: %[[LAB1:[0-9]+]] = lshr i32 %[[TEMP]], 3 +// MCDC: %[[LAB2:[0-9]+]] = zext i32 %[[LAB1]] to i64 +// MCDC: %[[LAB3:[0-9]+]] = add i64 ptrtoint (ptr @__profbm_test to i64), %[[LAB2]] +// MCDC: %[[LAB4:[0-9]+]] = inttoptr i64 %[[LAB3]] to ptr +// MCDC: %[[LAB5:[0-9]+]] = and i32 %[[TEMP]], 7 +// MCDC: %[[LAB6:[0-9]+]] = trunc i32 %[[LAB5]] to i8 +// MCDC: %[[LAB7:[0-9]+]] = shl i8 1, %[[LAB6]] +// MCDC: %mcdc.bits = load i8, ptr %[[LAB4]], align 1 +// MCDC: %[[LAB8:[0-9]+]] = or i8 %mcdc.bits, %[[LAB7]] +// MCDC: store i8 %[[LAB8]], ptr %[[LAB4]], align 1 diff --git a/compiler-rt/test/profile/ContinuousSyncMode/image-with-mcdc.c b/compiler-rt/test/profile/ContinuousSyncMode/image-with-mcdc.c new file mode 100644 index 000000000000..748af46ee52f --- /dev/null +++ b/compiler-rt/test/profile/ContinuousSyncMode/image-with-mcdc.c @@ -0,0 +1,26 @@ +// REQUIRES: darwin + +// RUN: %clang_profgen -fcoverage-mapping -fcoverage-mcdc -O3 -o %t.exe %s +// RUN: env LLVM_PROFILE_FILE="%c%t.profraw" %run %t.exe 3 3 +// RUN: llvm-profdata show --text --all-functions %t.profraw | FileCheck %s + +// CHECK: Num Bitmap Bytes: +// CHECK-NEXT: $1 +// CHECK-NEXT: Bitmap Byte Values: +// CHECK-NEXT: 8 +#include +#include +extern int __llvm_profile_is_continuous_mode_enabled(void); +int main(int argc, char *const argv[]) { + if (!__llvm_profile_is_continuous_mode_enabled()) + return 1; + + if (argc < 3) + return 1; + + if ((atoi(argv[1]) > 2) && (atoi(argv[2]) > 2)) { + printf("Decision Satisfied"); + } + + return 0; +} -- GitLab From 0b45c7722c7661063176f4dcd108ac4b6242fd34 Mon Sep 17 00:00:00 2001 From: madanial0 <118996571+madanial0@users.noreply.github.com> Date: Thu, 4 Jan 2024 14:18:43 -0500 Subject: [PATCH 025/728] [Flang] make ppc unsupported for x86_64 test case (NFC) (#73903) The test case is for x86_64 adding powerpc as unsupported. Co-authored-by: Mark Danial --- flang/test/Semantics/kinds04_q10.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flang/test/Semantics/kinds04_q10.f90 b/flang/test/Semantics/kinds04_q10.f90 index fbde6ed4c8cc..3da619d24dee 100644 --- a/flang/test/Semantics/kinds04_q10.f90 +++ b/flang/test/Semantics/kinds04_q10.f90 @@ -7,7 +7,7 @@ ! ! This test is for x86_64, where exponent-letter 'q' is for ! 10-byte extended precision -! UNSUPPORTED: system-windows +! UNSUPPORTED: system-windows, system-aix ! REQUIRES: x86-registered-target subroutine s(var) -- GitLab From 05216544a34eaf7aabf45df5f64d1c6c3e4e06c6 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Thu, 4 Jan 2024 11:25:09 -0800 Subject: [PATCH 026/728] [llvm] Add support for running tests as root (#75285) There are a few test that check access permissions, so they need to be disabled when running the tests as root. The most common use case for running tests as root is inside of a container. GitHub Actions, for example, only supports running the root user inside of containers, so this change is necessary in order to run the tests inside of a container running in the GitHub Actions environment. --- .../tools/llvm-ar/error-opening-permission.test | 1 + llvm/test/tools/llvm-dwarfdump/X86/output.s | 1 + llvm/test/tools/llvm-ifs/fail-file-write.test | 1 + .../llvm-ranlib/error-opening-permission.test | 1 + llvm/utils/lit/lit/llvm/config.py | 14 ++++++++++++++ 5 files changed, 18 insertions(+) diff --git a/llvm/test/tools/llvm-ar/error-opening-permission.test b/llvm/test/tools/llvm-ar/error-opening-permission.test index 4107bdfc044f..b42f95329a3c 100644 --- a/llvm/test/tools/llvm-ar/error-opening-permission.test +++ b/llvm/test/tools/llvm-ar/error-opening-permission.test @@ -1,6 +1,7 @@ ## Unsupported on windows as marking files "unreadable" ## is non-trivial on windows. # UNSUPPORTED: system-windows +# REQUIRES: non-root-user # RUN: rm -rf %t && mkdir -p %t # RUN: echo file1 > %t/1.txt diff --git a/llvm/test/tools/llvm-dwarfdump/X86/output.s b/llvm/test/tools/llvm-dwarfdump/X86/output.s index 37132eb55ca5..e7c9234ed74c 100644 --- a/llvm/test/tools/llvm-dwarfdump/X86/output.s +++ b/llvm/test/tools/llvm-dwarfdump/X86/output.s @@ -1,3 +1,4 @@ +# REQUIRES: non-root-user # RUN: rm -f %t1.txt %t2.txt %t3.txt # RUN: llvm-mc %S/brief.s -filetype obj -triple x86_64-apple-darwin -o %t.o diff --git a/llvm/test/tools/llvm-ifs/fail-file-write.test b/llvm/test/tools/llvm-ifs/fail-file-write.test index d5232070c1d0..f13500f22620 100644 --- a/llvm/test/tools/llvm-ifs/fail-file-write.test +++ b/llvm/test/tools/llvm-ifs/fail-file-write.test @@ -1,6 +1,7 @@ ## Test failing to write output file on non-windows platforms. # UNSUPPORTED: system-windows +# REQUIRES: non-root-user # RUN: rm -rf %t.TestDir # RUN: mkdir %t.TestDir # RUN: touch %t.TestDir/Output.TestFile diff --git a/llvm/test/tools/llvm-ranlib/error-opening-permission.test b/llvm/test/tools/llvm-ranlib/error-opening-permission.test index 1b1bb0def78d..be56962112e6 100644 --- a/llvm/test/tools/llvm-ranlib/error-opening-permission.test +++ b/llvm/test/tools/llvm-ranlib/error-opening-permission.test @@ -1,5 +1,6 @@ ## Unsupported on windows as marking files "unreadable" is non-trivial on windows. # UNSUPPORTED: system-windows +# REQUIRES: non-root-user # RUN: rm -rf %t && split-file %s %t && cd %t # RUN: yaml2obj 1.yaml -o 1.o diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py index 79094b839e77..8b0f2e6295ad 100644 --- a/llvm/utils/lit/lit/llvm/config.py +++ b/llvm/utils/lit/lit/llvm/config.py @@ -13,6 +13,17 @@ from lit.llvm.subst import ToolSubst lit_path_displayed = False +def user_is_root(): + # os.getuid() is not available on all platforms + try: + if os.getuid() == 0: + return True + except: + pass + + return False + + class LLVMConfig(object): def __init__(self, lit_config, config): self.lit_config = lit_config @@ -154,6 +165,9 @@ class LLVMConfig(object): if re.match(r'^ppc64le.*-linux', target_triple): features.add('target=powerpc64le-linux') + if not user_is_root(): + features.add("non-root-user") + use_gmalloc = lit_config.params.get("use_gmalloc", None) if lit.util.pythonize_bool(use_gmalloc): # Allow use of an explicit path for gmalloc library. -- GitLab From 852596d804a1b07c01e30f2091acf9c8bf16bda4 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Thu, 4 Jan 2024 11:29:00 -0800 Subject: [PATCH 027/728] [BasicAA] Guess reasonable contexts for separate storage hints (#76770) The definition of the pointer of the memory location being queried is always one such context. Even this conservative guess can be better than no guess at all in some cases. Fixes #64666 Co-authored-by: David Goldblatt --- llvm/include/llvm/Analysis/ValueTracking.h | 9 +++- llvm/lib/Analysis/BasicAliasAnalysis.cpp | 30 ++++++++++--- llvm/lib/Analysis/ValueTracking.cpp | 7 +-- .../BasicAA/separate_storage-alias-sets.ll | 43 +++++++++++++++++++ 4 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 llvm/test/Analysis/BasicAA/separate_storage-alias-sets.ll diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index baa16306ebf5..7360edfce1f3 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -810,9 +810,14 @@ bool isAssumeLikeIntrinsic(const Instruction *I); /// Return true if it is valid to use the assumptions provided by an /// assume intrinsic, I, at the point in the control-flow identified by the -/// context instruction, CxtI. +/// context instruction, CxtI. By default, ephemeral values of the assumption +/// are treated as an invalid context, to prevent the assumption from being used +/// to optimize away its argument. If the caller can ensure that this won't +/// happen, it can call with AllowEphemerals set to true to get more valid +/// assumptions. bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, - const DominatorTree *DT = nullptr); + const DominatorTree *DT = nullptr, + bool AllowEphemerals = false); enum class OverflowResult { /// Always overflows in the direction of signed/unsigned min value. diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp index 9eb7e914687c..a4a0846df7af 100644 --- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp @@ -1543,7 +1543,7 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, TLI, NullIsValidLocation))) return AliasResult::NoAlias; - if (CtxI && EnableSeparateStorageAnalysis) { + if (EnableSeparateStorageAnalysis) { for (AssumptionCache::ResultElem &Elem : AC.assumptionsFor(O1)) { if (!Elem || Elem.Index == AssumptionCache::ExprResultIdx) continue; @@ -1559,10 +1559,30 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, const Value *HintO1 = getUnderlyingObject(Hint1); const Value *HintO2 = getUnderlyingObject(Hint2); - if (((O1 == HintO1 && O2 == HintO2) || - (O1 == HintO2 && O2 == HintO1)) && - isValidAssumeForContext(Assume, CtxI, DT)) - return AliasResult::NoAlias; + auto ValidAssumeForPtrContext = [&](const Value *Ptr) { + if (const Instruction *PtrI = dyn_cast(Ptr)) { + return isValidAssumeForContext(Assume, PtrI, DT, + /* AllowEphemerals */ true); + } + if (const Argument *PtrA = dyn_cast(Ptr)) { + const Instruction *FirstI = + &*PtrA->getParent()->getEntryBlock().begin(); + return isValidAssumeForContext(Assume, FirstI, DT, + /* AllowEphemerals */ true); + } + return false; + }; + + if ((O1 == HintO1 && O2 == HintO2) || (O1 == HintO2 && O2 == HintO1)) { + // Note that we go back to V1 and V2 for the + // ValidAssumeForPtrContext checks; they're dominated by O1 and O2, + // so strictly more assumptions are valid for them. + if ((CtxI && isValidAssumeForContext(Assume, CtxI, DT, + /* AllowEphemerals */ true)) || + ValidAssumeForPtrContext(V1) || ValidAssumeForPtrContext(V2)) { + return AliasResult::NoAlias; + } + } } } } diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 439127e5c954..940ae9eb7ee2 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -485,7 +485,8 @@ bool llvm::isAssumeLikeIntrinsic(const Instruction *I) { bool llvm::isValidAssumeForContext(const Instruction *Inv, const Instruction *CxtI, - const DominatorTree *DT) { + const DominatorTree *DT, + bool AllowEphemerals) { // There are two restrictions on the use of an assume: // 1. The assume must dominate the context (or the control flow must // reach the assume whenever it reaches the context). @@ -503,7 +504,7 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv, // Don't let an assume affect itself - this would cause the problems // `isEphemeralValueOf` is trying to prevent, and it would also make // the loop below go out of bounds. - if (Inv == CxtI) + if (!AllowEphemerals && Inv == CxtI) return false; // The context comes first, but they're both in the same block. @@ -516,7 +517,7 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv, if (!isGuaranteedToTransferExecutionToSuccessor(Range, 15)) return false; - return !isEphemeralValueOf(Inv, CxtI); + return AllowEphemerals || !isEphemeralValueOf(Inv, CxtI); } // Inv and CxtI are in different blocks. diff --git a/llvm/test/Analysis/BasicAA/separate_storage-alias-sets.ll b/llvm/test/Analysis/BasicAA/separate_storage-alias-sets.ll new file mode 100644 index 000000000000..7bd87d1b227b --- /dev/null +++ b/llvm/test/Analysis/BasicAA/separate_storage-alias-sets.ll @@ -0,0 +1,43 @@ +; We want BasicAA to make a reasonable conservative guess as to context for +; separate storage hints. This lets alias analysis users (such as the alias set +; tracker) who can't be context-sensitive still get the benefits of hints. + +; RUN: opt < %s -basic-aa-separate-storage -S -passes=print-alias-sets 2>&1 | FileCheck %s + +declare void @llvm.assume(i1) + +; CHECK-LABEL: Alias sets for function 'arg_arg' +; CHECK: AliasSet[{{.*}}, 1] must alias, Ref Pointers: (ptr %a1, LocationSize::precise(1)) +; CHECK: AliasSet[{{.*}}, 1] must alias, Ref Pointers: (ptr %a2, LocationSize::precise(1)) +define void @arg_arg(ptr %a1, ptr %a2) { +entry: + call void @llvm.assume(i1 true) [ "separate_storage"(ptr %a1, ptr %a2) ] + %0 = load i8, ptr %a1 + %1 = load i8, ptr %a2 + ret void +} + +; CHECK-LABEL: Alias sets for function 'arg_inst' +; CHECK: AliasSet[{{.*}}, 1] must alias, Ref Pointers: (ptr %a1, LocationSize::precise(1)) +; CHECK: AliasSet[{{.*}}, 1] must alias, Ref Pointers: (ptr %0, LocationSize::precise(1)) +define void @arg_inst(ptr %a1, ptr %a2) { +entry: + %0 = getelementptr inbounds i8, ptr %a2, i64 20 + call void @llvm.assume(i1 true) [ "separate_storage"(ptr %a1, ptr %0) ] + %1 = load i8, ptr %a1 + %2 = load i8, ptr %0 + ret void +} + +; CHECK-LABEL: Alias sets for function 'inst_inst' +; CHECK: AliasSet[{{.*}}, 1] must alias, Ref Pointers: (ptr %0, LocationSize::precise(1)) +; CHECK: AliasSet[{{.*}}, 1] must alias, Ref Pointers: (ptr %1, LocationSize::precise(1)) +define void @inst_inst(ptr %a1, ptr %a2) { +entry: + %0 = getelementptr inbounds i8, ptr %a1, i64 20 + %1 = getelementptr inbounds i8, ptr %a2, i64 20 + call void @llvm.assume(i1 true) [ "separate_storage"(ptr %0, ptr %1) ] + %2 = load i8, ptr %0 + %3 = load i8, ptr %1 + ret void +} -- GitLab From 8f40783944fcdd33a2ad134be0d26a671202fd85 Mon Sep 17 00:00:00 2001 From: Felipe de Azevedo Piovezan Date: Thu, 4 Jan 2024 16:30:49 -0300 Subject: [PATCH 028/728] [lldb][nfc] Mark function as const (#76974) This function has no mutable behavior --- lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp | 2 +- lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp index 7c253553d57b..b718f98340a7 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp @@ -48,7 +48,7 @@ DebugNamesDWARFIndex::GetUnits(const DebugNames &debug_names) { } std::optional -DebugNamesDWARFIndex::ToDIERef(const DebugNames::Entry &entry) { +DebugNamesDWARFIndex::ToDIERef(const DebugNames::Entry &entry) const { // Look for a DWARF unit offset (CU offset or local TU offset) as they are // both offsets into the .debug_info section. std::optional unit_offset = entry.getCUOffset(); diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h index 7ce630a56137..cca0913c4124 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h @@ -79,7 +79,7 @@ private: std::unique_ptr m_debug_names_up; ManualDWARFIndex m_fallback; - std::optional ToDIERef(const DebugNames::Entry &entry); + std::optional ToDIERef(const DebugNames::Entry &entry) const; bool ProcessEntry(const DebugNames::Entry &entry, llvm::function_ref callback); -- GitLab From 166bd4e1f18da221621953bd5943c1a8d17201fe Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Thu, 4 Jan 2024 11:30:57 -0800 Subject: [PATCH 029/728] [workflows] Build a container for running CI on github actions (#75286) Using a container will allow us to have similar testing environments on both the GitHub hosted runners and the self-hosted runners. --- .github/workflows/build-ci-container.yml | 60 +++++++++++++++++++ .../containers/github-action-ci/Dockerfile | 48 +++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 .github/workflows/build-ci-container.yml create mode 100644 .github/workflows/containers/github-action-ci/Dockerfile diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml new file mode 100644 index 000000000000..ad3d50d4d578 --- /dev/null +++ b/.github/workflows/build-ci-container.yml @@ -0,0 +1,60 @@ + +name: Build CI Container + +permissions: + contents: read + +on: + push: + branches: + - main + paths: + - .github/workflows/build-ci-container.yml + - '.github/workflows/containers/github-action-ci/**' + pull_request: + branches: + - main + paths: + - .github/workflows/build-ci-container.yml + - '.github/workflows/containers/github-action-ci/**' + +jobs: + build-ci-container: + if: github.repository_owner == 'llvm' + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Write Variables + id: vars + run: | + tag=`date +%s` + container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/ci-ubuntu-22.04" + echo "container-name=$container_name" >> $GITHUB_OUTPUT + echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT + + - name: Checkout LLVM + uses: actions/checkout@v4 + with: + sparse-checkout: .github/workflows/containers/github-action-ci/ + + - name: Build Container + working-directory: ./.github/workflows/containers/github-action-ci/ + run: | + podman build -t ${{ steps.vars.outputs.container-name-tag }} . + podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest + + - name: Test Container + run: | + for image in ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}; do + podman run --rm -it $image /usr/bin/bash -x -c 'printf '\''#include \nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello' + done + + - name: Push Container + if: github.event_name == 'push' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io + podman push ${{ steps.vars.outputs.container-name-tag }} + podman push ${{ steps.vars.outputs.container-name }}:latest diff --git a/.github/workflows/containers/github-action-ci/Dockerfile b/.github/workflows/containers/github-action-ci/Dockerfile new file mode 100644 index 000000000000..d91a7ad3a9d0 --- /dev/null +++ b/.github/workflows/containers/github-action-ci/Dockerfile @@ -0,0 +1,48 @@ +FROM docker.io/library/ubuntu:22.04 as base +ENV LLVM_SYSROOT=/opt/llvm/ + +FROM base as toolchain +ENV LLVM_MAJOR=17 +ENV LLVM_VERSION=${LLVM_MAJOR}.0.6 +ENV LLVM_DIRNAME=clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-22.04 +ENV LLVM_FILENAME=${LLVM_DIRNAME}.tar.xz + +RUN apt-get update && \ + apt-get install -y \ + curl \ + xz-utils + +RUN mkdir -p $LLVM_SYSROOT/bin/ $LLVM_SYSROOT/lib/ + +RUN curl -O -L https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/$LLVM_FILENAME + +RUN tar -C $LLVM_SYSROOT --strip-components=1 -xJf $LLVM_FILENAME \ + $LLVM_DIRNAME/bin/clang \ + $LLVM_DIRNAME/bin/clang++ \ + $LLVM_DIRNAME/bin/clang-cl \ + $LLVM_DIRNAME/bin/clang-$LLVM_MAJOR \ + $LLVM_DIRNAME/bin/lld \ + $LLVM_DIRNAME/bin/ld.lld \ + $LLVM_DIRNAME/lib/clang/ + + +FROM base + +COPY --from=toolchain $LLVM_SYSROOT $LLVM_SYSROOT + +# Need to install curl for hendrikmuhs/ccache-action +# Need nodejs for some of the GitHub actions. +# Need perl-modules for clang analyzer tests. +RUN apt-get update && \ + apt-get install -y \ + binutils \ + cmake \ + curl \ + libstdc++-11-dev \ + ninja-build \ + nodejs \ + perl-modules \ + python3-psutil + +ENV LLVM_SYSROOT=$LLVM_SYSROOT +ENV PATH=${LLVM_SYSROOT}/bin:${PATH} -- GitLab From 45651c357ae4e39a115bae6a936151b1741abf70 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 4 Jan 2024 11:56:32 -0800 Subject: [PATCH 030/728] [RISCV] Fix indentation in riscv_sifive_vector.td. NFC --- clang/include/clang/Basic/riscv_sifive_vector.td | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Basic/riscv_sifive_vector.td b/clang/include/clang/Basic/riscv_sifive_vector.td index 0d471f6c554c..ef5114d6105e 100644 --- a/clang/include/clang/Basic/riscv_sifive_vector.td +++ b/clang/include/clang/Basic/riscv_sifive_vector.td @@ -121,11 +121,11 @@ multiclass RVVVQMACCDODBuiltinSet> suffixes_prototypes> { } multiclass RVVVQMACCQOQBuiltinSet> suffixes_prototypes> { - let OverloadedName = NAME, - Name = NAME, - HasMasked = false, - Log2LMUL = [-1, 0, 1, 2] in - defm NAME : RVVOutOp1Op2BuiltinSet; + let OverloadedName = NAME, + Name = NAME, + HasMasked = false, + Log2LMUL = [-1, 0, 1, 2] in + defm NAME : RVVOutOp1Op2BuiltinSet; } multiclass RVVVFNRCLIPBuiltinSet { -- GitLab From b5a3e9639291359d5c9e16f0610393d92bc7d4c2 Mon Sep 17 00:00:00 2001 From: ChipsSpectre Date: Thu, 4 Jan 2024 21:04:54 +0100 Subject: [PATCH 031/728] [Clang][Parser] Fix crash of clang when using C++ constructs like :: in C code (#74926) Ensure we do not try to parse a nested-name-specifier when parsing an ill-formed file in C mode. Fixes #73559 --- clang/docs/ReleaseNotes.rst | 6 ++++-- clang/lib/Parse/ParseDecl.cpp | 3 ++- clang/lib/Parse/ParseDeclCXX.cpp | 2 ++ clang/test/Parser/cxx-in-c.c | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index e5de042cebd4..ce7599ad34be 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -686,9 +686,11 @@ Bug Fixes in This Version (`#65568 `_) - Fix an issue where clang doesn't respect detault template arguments that are added in a later redeclaration for CTAD. - Fixes (#69987 `_) + Fixes (`#69987 `_) - Fix an issue where CTAD fails for explicit type conversion. - Fixes (#64347 `_) + Fixes (`#64347 `_) +- Fix crash when using C++ only tokens like ``::`` in C compiler clang. + Fixes (`#73559 `_) Bug Fixes to Compiler Builtins diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index ed006f9d67de..b60ae293ef8c 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -3483,7 +3483,8 @@ void Parser::ParseDeclarationSpecifiers( case tok::coloncolon: // ::foo::bar // C++ scope specifier. Annotate and loop, or bail out on error. - if (TryAnnotateCXXScopeToken(EnteringContext)) { + if (getLangOpts().CPlusPlus && + TryAnnotateCXXScopeToken(EnteringContext)) { if (!DS.hasTypeSpecifier()) DS.SetTypeSpecError(); goto DoneWithDeclSpec; diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index 910112ecae96..d97081da4200 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -2679,6 +2679,8 @@ Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS, ParsedAttributes &AccessAttrs, const ParsedTemplateInfo &TemplateInfo, ParsingDeclRAIIObject *TemplateDiags) { + assert(getLangOpts().CPlusPlus && + "ParseCXXClassMemberDeclaration should only be called in C++ mode"); if (Tok.is(tok::at)) { if (getLangOpts().ObjC && NextToken().isObjCAtKeyword(tok::objc_defs)) Diag(Tok, diag::err_at_defs_cxx); diff --git a/clang/test/Parser/cxx-in-c.c b/clang/test/Parser/cxx-in-c.c index f5fa39bd0cb9..034a44cdf12b 100644 --- a/clang/test/Parser/cxx-in-c.c +++ b/clang/test/Parser/cxx-in-c.c @@ -3,3 +3,6 @@ // PR9137 void f0(int x) : {}; // expected-error{{expected function body after function declarator}} void f1(int x) try {}; // expected-error{{expected function body after function declarator}} + +// GH73559 +::; // expected-error{{expected identifier or '('}} -- GitLab From 9d829784d4c2b4f82e0b301eaa840c9ed192e919 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray Date: Thu, 4 Jan 2024 20:06:47 +0000 Subject: [PATCH 032/728] [AArch64] Correct features for Arm Cortex-A78C, Cortex-X1C and Cortex-X2 (#76932) Remove AArch64::AEK_FP16ML from Arm Cortex-A78C definition, as this is not supported, according to the Technical Reference Manual: https://developer.arm.com/documentation/102226/latest/ Also add AArch64::AEK_FLAGM (Flag Manipulation) to Arm Cortex-X1C and Arm Cortex-X2 as these were missing previously, but are supported, according to the Technical Reference Manuals: https://developer.arm.com/documentation/101968/latest/ https://developer.arm.com/documentation/101803/latest/ Fixes #62383 --- llvm/include/llvm/TargetParser/AArch64TargetParser.h | 7 +++---- llvm/lib/Target/AArch64/AArch64.td | 2 +- llvm/unittests/TargetParser/TargetParserTest.cpp | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 53dc2be825f2..2fe4d5eeb742 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -482,8 +482,7 @@ inline constexpr CpuInfo CpuInfos[] = { (AArch64::ExtensionBitset( {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_FP16, AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS, - AArch64::AEK_PROFILE, AArch64::AEK_FLAGM, AArch64::AEK_PAUTH, - AArch64::AEK_FP16FML}))}, + AArch64::AEK_PROFILE, AArch64::AEK_FLAGM, AArch64::AEK_PAUTH}))}, {"cortex-a710", ARMV9A, (AArch64::ExtensionBitset( {AArch64::AEK_MTE, AArch64::AEK_PAUTH, AArch64::AEK_FLAGM, @@ -514,13 +513,13 @@ inline constexpr CpuInfo CpuInfos[] = { (AArch64::ExtensionBitset( {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_FP16, AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS, - AArch64::AEK_PAUTH, AArch64::AEK_PROFILE}))}, + AArch64::AEK_PAUTH, AArch64::AEK_PROFILE, AArch64::AEK_FLAGM}))}, {"cortex-x2", ARMV9A, (AArch64::ExtensionBitset( {AArch64::AEK_MTE, AArch64::AEK_BF16, AArch64::AEK_I8MM, AArch64::AEK_PAUTH, AArch64::AEK_SSBS, AArch64::AEK_SB, AArch64::AEK_SVE, AArch64::AEK_SVE2, AArch64::AEK_SVE2BITPERM, - AArch64::AEK_FP16FML}))}, + AArch64::AEK_FP16FML, AArch64::AEK_FLAGM}))}, {"cortex-x3", ARMV9A, (AArch64::ExtensionBitset( {AArch64::AEK_SVE, AArch64::AEK_PERFMON, AArch64::AEK_PROFILE, diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td index 68f452039c9b..d5e8ed101d1c 100644 --- a/llvm/lib/Target/AArch64/AArch64.td +++ b/llvm/lib/Target/AArch64/AArch64.td @@ -1405,7 +1405,7 @@ def ProcessorFeatures { FeatureSSBS]; list A78C = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8, FeatureNEON, FeatureFullFP16, FeatureDotProd, - FeatureFlagM, FeatureFP16FML, FeaturePAuth, + FeatureFlagM, FeaturePAuth, FeaturePerfMon, FeatureRCPC, FeatureSPE, FeatureSSBS]; list A710 = [HasV9_0aOps, FeatureNEON, FeaturePerfMon, diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index 92bd4da1d3a4..6b2cbb33ba8d 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -1197,7 +1197,7 @@ INSTANTIATE_TEST_SUITE_P( AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_RDM, AArch64::AEK_FP16, AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS, AArch64::AEK_PROFILE, AArch64::AEK_FLAGM, - AArch64::AEK_PAUTH, AArch64::AEK_FP16FML})), + AArch64::AEK_PAUTH})), "8.2-A"), ARMCPUTestParams( "cortex-a710", "armv9-a", "neon-fp-armv8", @@ -1289,7 +1289,7 @@ INSTANTIATE_TEST_SUITE_P( AArch64::AEK_FP, AArch64::AEK_RDM, AArch64::AEK_SIMD, AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_FP16, AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS, - AArch64::AEK_PAUTH, AArch64::AEK_PROFILE})), + AArch64::AEK_PAUTH, AArch64::AEK_PROFILE, AArch64::AEK_FLAGM})), "8.2-A"), ARMCPUTestParams( "cortex-x2", "armv9-a", "neon-fp-armv8", @@ -1300,7 +1300,7 @@ INSTANTIATE_TEST_SUITE_P( AArch64::AEK_PAUTH, AArch64::AEK_I8MM, AArch64::AEK_BF16, AArch64::AEK_SVE, AArch64::AEK_SVE2, AArch64::AEK_SVE2BITPERM, AArch64::AEK_SSBS, AArch64::AEK_SB, AArch64::AEK_FP16, - AArch64::AEK_FP16FML})), + AArch64::AEK_FP16FML, AArch64::AEK_FLAGM})), "9-A"), ARMCPUTestParams( "cortex-x3", "armv9-a", "neon-fp-armv8", -- GitLab From cd3942059eed7b7185f26bc583ac287a995db0d0 Mon Sep 17 00:00:00 2001 From: Krzysztof Drewniak Date: Thu, 4 Jan 2024 14:08:30 -0600 Subject: [PATCH 033/728] [SeperateConstOffsetFromGEP] Pre-commit tests for or disjoint handling (#76972) 1. Adds tests for the existing interpretation of `or` as `add` in SeperateConstOffsetFromGEP. 2. Pre-commits a test for `or disjoint`. --- .../split-gep-or-as-add.ll | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/split-gep-or-as-add.ll diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/split-gep-or-as-add.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/split-gep-or-as-add.ll new file mode 100644 index 000000000000..45154f5a68f9 --- /dev/null +++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/split-gep-or-as-add.ll @@ -0,0 +1,61 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes="separate-const-offset-from-gep" < %s | FileCheck %s + +;; Check that or operations, either with operands with no bits in common or that +;; are disjoint are lowered into constant GEPs. Note that because this is a +;; target-independent test, the GEP seperator will lower the seperated-off constant +;; part to ptrtoint-based arithmetic. + +define void @testOrDoesntSplit(ptr %p) { +; CHECK-LABEL: define void @testOrDoesntSplit( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[VAR:%.*]] = tail call i64 @foo() +; CHECK-NEXT: [[OFF:%.*]] = or i64 [[VAR]], 10 +; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 [[OFF]] +; CHECK-NEXT: store i8 0, ptr [[Q]], align 1 +; CHECK-NEXT: ret void +; + %var = tail call i64 @foo() + %off = or i64 %var, 10 + %q = getelementptr i8, ptr %p, i64 %off + store i8 0, ptr %q + ret void +} + +define void @testNoBitsInCommonOrSplits(ptr %p) { +; CHECK-LABEL: define void @testNoBitsInCommonOrSplits( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[VAR:%.*]] = tail call i64 @foo() +; CHECK-NEXT: [[VAR_HIGH:%.*]] = and i64 [[VAR]], -16 +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], [[VAR_HIGH]] +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 10 +; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr +; CHECK-NEXT: store i8 0, ptr [[TMP4]], align 1 +; CHECK-NEXT: ret void +; + %var = tail call i64 @foo() + %var.high = and i64 %var, -16 + %off = or i64 %var.high, 10 + %q = getelementptr i8, ptr %p, i64 %off + store i8 0, ptr %q + ret void +} + +define void @testDisjointOrSplits(ptr %p) { +; CHECK-LABEL: define void @testDisjointOrSplits( +; CHECK-SAME: ptr [[P:%.*]]) { +; CHECK-NEXT: [[VAR:%.*]] = tail call i64 @foo() +; CHECK-NEXT: [[OFF:%.*]] = or disjoint i64 [[VAR]], 10 +; CHECK-NEXT: [[Q:%.*]] = getelementptr i8, ptr [[P]], i64 [[OFF]] +; CHECK-NEXT: store i8 0, ptr [[Q]], align 1 +; CHECK-NEXT: ret void +; + %var = tail call i64 @foo() + %off = or disjoint i64 %var, 10 + %q = getelementptr i8, ptr %p, i64 %off + store i8 0, ptr %q + ret void +} + +declare i64 @foo() -- GitLab From c398923f32ff5373964e1288d9de37403c2399dd Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Thu, 4 Jan 2024 12:14:32 -0800 Subject: [PATCH 034/728] [CMake][runtimes] Check LLVM_ENABLE_PROJECTS for libc (#76845) Only some targets may be building llvm-libc in which case the top-level LLVM_ENABLE_RUNTIMES variable won't contain libc. Rather, we can check LLVM_ENABLE_PROJECTS since libc is required to be included there for libc-hdrgen to be built (and when LLVM_ENABLE_RUNTIMES contains libc, we automatically include libc in LLVM_ENABLE_PROJECTS as well). --- llvm/runtimes/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt index db50b6d1e8f8..b9acb862cc5c 100644 --- a/llvm/runtimes/CMakeLists.txt +++ b/llvm/runtimes/CMakeLists.txt @@ -423,7 +423,7 @@ if(runtimes) endif() endforeach() endif() - if("libc" IN_LIST LLVM_ENABLE_RUNTIMES AND + if("libc" IN_LIST LLVM_ENABLE_PROJECTS AND (LLVM_LIBC_FULL_BUILD OR LIBC_GPU_BUILD OR LIBC_GPU_ARCHITECTURES)) if(TARGET libc-hdrgen) set(libc_tools libc-hdrgen) -- GitLab From 71c17424b5d5c22c0ce6b4c41acaa0401515baca Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Thu, 4 Jan 2024 21:33:51 +0100 Subject: [PATCH 035/728] [mlir][TD] update more tests to use the "main" interpreter pass (#76963) Update several tests under mlir/test/Dialect/Transform to use the "main" transform interpreter pass with named entry points rather than the test interpreter pass. This helped discover a logic error in the expensive checks mechanism that was exiting too early. --- .../Transform/IR/TransformInterfaces.cpp | 7 +- .../Transform/apply-foreach-nested.mlir | 26 +- .../Dialect/Transform/expensive-checks.mlir | 395 +-- .../Transform/selective-targeting.mlir | 104 +- .../Dialect/Transform/test-interpreter.mlir | 2127 +++++++++-------- 5 files changed, 1464 insertions(+), 1195 deletions(-) diff --git a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp b/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp index cd66a0e566f6..371ad904dcae 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp @@ -629,9 +629,6 @@ void transform::TransformState::recordOpHandleInvalidation( // the IR nested in each payload op associated with the given handle and look // for handles associated with each operation and value. for (const auto &[region, mapping] : llvm::reverse(mappings)) { - // Stop lookup when reaching a region that is isolated from above. - if (region->getParentOp()->hasTrait()) - break; // Go over all op handle mappings and mark as invalidated any handle // pointing to any of the payload ops associated with the given handle or // any op nested in them. @@ -652,6 +649,10 @@ void transform::TransformState::recordOpHandleInvalidation( payloadValue, valueHandle, newlyInvalidated); } + + // Stop lookup when reaching a region that is isolated from above. + if (region->getParentOp()->hasTrait()) + break; } } diff --git a/mlir/test/Dialect/Transform/apply-foreach-nested.mlir b/mlir/test/Dialect/Transform/apply-foreach-nested.mlir index d2f71644bbc5..23f45b97bc2f 100644 --- a/mlir/test/Dialect/Transform/apply-foreach-nested.mlir +++ b/mlir/test/Dialect/Transform/apply-foreach-nested.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s --split-input-file --verify-diagnostics \ -// RUN: --pass-pipeline="builtin.module(test-transform-dialect-interpreter{enable-expensive-checks=1 bind-first-extra-to-ops=scf.for})" +// RUN: --transform-interpreter func.func private @bar() @@ -17,12 +17,15 @@ func.func @foo() { return } -transform.sequence failures(suppress) { -^bb0(%arg0: !transform.any_op, %arg1: !transform.op<"scf.for">): - %1 = transform.test_reverse_payload_ops %arg1 : (!transform.op<"scf.for">) -> !transform.op<"scf.for"> - // expected-error @below {{transform operation consumes a handle pointing to an ancestor payload operation before its descendant}} - // expected-note @below {{the ancestor is likely erased or rewritten before the descendant is accessed, leading to undefined behavior}} - transform.test_consume_operand_each %1 : !transform.op<"scf.for"> +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.structured.match ops{["scf.for"]} in %arg0 : (!transform.any_op) -> !transform.op<"scf.for"> + %1 = transform.test_reverse_payload_ops %0 : (!transform.op<"scf.for">) -> !transform.op<"scf.for"> + // expected-error @below {{transform operation consumes a handle pointing to an ancestor payload operation before its descendant}} + // expected-note @below {{the ancestor is likely erased or rewritten before the descendant is accessed, leading to undefined behavior}} + transform.test_consume_operand_each %1 : !transform.op<"scf.for"> + transform.yield + } } // ----- @@ -42,7 +45,10 @@ func.func @foo() { } // No error here, processing ancestors before descendants. -transform.sequence failures(suppress) { -^bb0(%arg0: !transform.any_op, %arg1: !transform.op<"scf.for">): - transform.test_consume_operand_each %arg1 : !transform.op<"scf.for"> +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.structured.match ops{["scf.for"]} in %arg0 : (!transform.any_op) -> !transform.op<"scf.for"> + transform.test_consume_operand_each %0 : !transform.op<"scf.for"> + transform.yield + } } diff --git a/mlir/test/Dialect/Transform/expensive-checks.mlir b/mlir/test/Dialect/Transform/expensive-checks.mlir index ee9a5af80552..b532e1892b1a 100644 --- a/mlir/test/Dialect/Transform/expensive-checks.mlir +++ b/mlir/test/Dialect/Transform/expensive-checks.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt --test-transform-dialect-interpreter='enable-expensive-checks=1' --split-input-file --verify-diagnostics %s +// RUN: mlir-opt --transform-interpreter --split-input-file --verify-diagnostics %s // expected-note @below {{ancestor payload op}} func.func @func() { @@ -6,24 +6,29 @@ func.func @func() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @return : benefit(1) { - %0 = operands - %1 = types - %2 = operation "func.return"(%0 : !pdl.range) -> (%1 : !pdl.range) - rewrite %2 with "transform.dialect" - } - - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - // expected-note @below {{handle to invalidated ops}} - %0 = pdl_match @return in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - // expected-note @below {{invalidated by this transform op that consumes its operand #0}} - test_consume_operand %1 : !transform.any_op - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_print_remark_at_operand %0, "remark" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @return : benefit(1) { + %0 = operands + %1 = types + %2 = operation "func.return"(%0 : !pdl.range) -> (%1 : !pdl.range) + rewrite %2 with "transform.dialect" + } + + sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + // expected-note @below {{handle to invalidated ops}} + %0 = pdl_match @return in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op + // expected-note @below {{invalidated by this transform op that consumes its operand #0}} + test_consume_operand %1 : !transform.any_op + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + test_print_remark_at_operand %0, "remark" : !transform.any_op + } + } + transform.yield } } @@ -35,29 +40,34 @@ func.func @func1() { } func.func private @func2() -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @func : benefit(1) { - %0 = operands - %1 = types - %2 = operation "func.func"(%0 : !pdl.range) -> (%1 : !pdl.range) - rewrite %2 with "transform.dialect" - } - pdl.pattern @return : benefit(1) { - %0 = operands - %1 = types - %2 = operation "func.return"(%0 : !pdl.range) -> (%1 : !pdl.range) - rewrite %2 with "transform.dialect" - } - - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @func in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = pdl_match @return in %arg1 : (!transform.any_op) -> !transform.any_op - %2 = replicate num(%0) %1 : !transform.any_op, !transform.any_op - // expected-error @below {{a handle passed as operand #0 and consumed by this operation points to a payload entity more than once}} - test_consume_operand %2 : !transform.any_op - test_print_remark_at_operand %0, "remark" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @func : benefit(1) { + %0 = operands + %1 = types + %2 = operation "func.func"(%0 : !pdl.range) -> (%1 : !pdl.range) + rewrite %2 with "transform.dialect" + } + pdl.pattern @return : benefit(1) { + %0 = operands + %1 = types + %2 = operation "func.return"(%0 : !pdl.range) -> (%1 : !pdl.range) + rewrite %2 with "transform.dialect" + } + + sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @func in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = pdl_match @return in %arg1 : (!transform.any_op) -> !transform.any_op + %2 = replicate num(%0) %1 : !transform.any_op, !transform.any_op + // expected-error @below {{a handle passed as operand #0 and consumed by this operation points to a payload entity more than once}} + test_consume_operand %2 : !transform.any_op + test_print_remark_at_operand %0, "remark" : !transform.any_op + } + } + transform.yield } } @@ -66,10 +76,9 @@ transform.with_pdl_patterns { // expected-note @below {{ancestor payload op}} // expected-note @below {{nested payload op}} -module { +module attributes {transform.with_named_sequence} { - transform.sequence failures(propagate) { - ^bb0(%0: !transform.any_op): + transform.named_sequence @__transform_main(%0: !transform.any_op) { %1 = transform.test_copy_payload %0 : (!transform.any_op) -> !transform.any_op // expected-note @below {{handle to invalidated ops}} %2 = transform.test_copy_payload %0 : (!transform.any_op) ->!transform.any_op @@ -77,6 +86,7 @@ module { transform.test_consume_operand %1 : !transform.any_op // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} transform.test_consume_operand %2 : !transform.any_op + transform.yield } } @@ -84,10 +94,9 @@ module { // expected-note @below {{ancestor payload op}} // expected-note @below {{nested payload op}} -module { +module attributes {transform.with_named_sequence} { - transform.sequence failures(propagate) { - ^bb0(%0: !transform.any_op): + transform.named_sequence @__transform_main(%0: !transform.any_op) { %1 = transform.test_copy_payload %0 : (!transform.any_op) -> !transform.any_op // expected-note @below {{handle to invalidated ops}} %2 = transform.test_copy_payload %0 : (!transform.any_op) -> !transform.any_op @@ -97,6 +106,7 @@ module { // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities}} transform.test_consume_operand %1, %2 : !transform.any_op, !transform.any_op + transform.yield } } @@ -104,13 +114,13 @@ module { // Deduplication attribute allows "merge_handles" to take repeated operands. -module { +module attributes {transform.with_named_sequence} { - transform.sequence failures(propagate) { - ^bb0(%0: !transform.any_op): + transform.named_sequence @__transform_main(%0: !transform.any_op) { %1 = transform.test_copy_payload %0 : (!transform.any_op) -> !transform.any_op %2 = transform.test_copy_payload %0 : (!transform.any_op) -> !transform.any_op transform.merge_handles %1, %2 { deduplicate } : !transform.any_op + transform.yield } } // ----- @@ -118,16 +128,18 @@ module { // expected-note @below {{payload value}} %0 = "test.match_anchor"() : () -> (i32) -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated handle}} - %4 = test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates handles to the same values as associated with it}} - test_consume_operand %3 : !transform.any_value - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %4 : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated handle}} + %4 = transform.test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates handles to the same values as associated with it}} + transform.test_consume_operand %3 : !transform.any_value + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %4 : !transform.any_value + transform.yield + } } // ----- @@ -137,15 +149,17 @@ transform.sequence failures(propagate) { // expected-note @below {{op defining the value as result #0}} %0 = "test.match_anchor"() : () -> (i32) -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op - // expected-note @below {{invalidated handle}} - %3 = test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %2 : !transform.any_op - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %3 : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op + // expected-note @below {{invalidated handle}} + %3 = transform.test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %2 : !transform.any_op + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %3 : !transform.any_value + transform.yield + } } // ----- @@ -159,16 +173,18 @@ transform.sequence failures(propagate) { "test.region_terminator"() : () -> () }) : () -> () -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op - // expected-note @below {{invalidated handle}} - %3 = test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %1 : !transform.any_op - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %3 : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op + // expected-note @below {{invalidated handle}} + %3 = transform.test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %1 : !transform.any_op + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %3 : !transform.any_value + transform.yield + } } // ----- @@ -182,16 +198,18 @@ transform.sequence failures(propagate) { "test.region_terminator"() : () -> () }) : () -> () -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op - // expected-note @below {{invalidated handle}} - %3 = test_produce_value_handle_to_argument_of_parent_block %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %1 : !transform.any_op - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %3 : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op + // expected-note @below {{invalidated handle}} + %3 = transform.test_produce_value_handle_to_argument_of_parent_block %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %1 : !transform.any_op + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %3 : !transform.any_value + transform.yield + } } // ----- @@ -208,16 +226,18 @@ transform.sequence failures(propagate) { }): () -> () }) : () -> () -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op - // expected-note @below {{invalidated handle}} - %3 = test_produce_value_handle_to_argument_of_parent_block %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %1 : !transform.any_op - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %3 : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op + // expected-note @below {{invalidated handle}} + %3 = transform.test_produce_value_handle_to_argument_of_parent_block %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %1 : !transform.any_op + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %3 : !transform.any_value + transform.yield + } } // ----- @@ -227,15 +247,17 @@ transform.sequence failures(propagate) { // expected-note @below {{consumed handle points to this payload value}} %0 = "test.match_anchor"() : () -> (i32) -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - // expected-note @below {{handle to invalidated ops}} - %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %3 : !transform.any_value - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %2 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-note @below {{handle to invalidated ops}} + %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_result %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %3 : !transform.any_value + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %2 : !transform.any_op + transform.yield + } } // ----- @@ -249,16 +271,18 @@ transform.sequence failures(propagate) { "test.region_terminator"() : () -> () }) : () -> (i32) -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op - // expected-note @below {{handle to invalidated ops}} - %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_result %1, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %3 : !transform.any_value - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %2 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op + // expected-note @below {{handle to invalidated ops}} + %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_result %1, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %3 : !transform.any_value + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %2 : !transform.any_op + transform.yield + } } @@ -273,15 +297,17 @@ transform.sequence failures(propagate) { "test.region_terminator"() : () -> () }) : () -> () -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - // expected-note @below {{handle to invalidated ops}} - %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_argument_of_parent_block %2, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %3 : !transform.any_value - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %2 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-note @below {{handle to invalidated ops}} + %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_argument_of_parent_block %2, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %3 : !transform.any_value + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %2 : !transform.any_op + transform.yield + } } // ----- @@ -299,16 +325,18 @@ transform.sequence failures(propagate) { "test.match_anchor_1"() : () -> () }) : () -> () -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op - // expected-note @below {{handle to invalidated ops}} - %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_argument_of_parent_block %1, 0 : (!transform.any_op) -> !transform.any_value - // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} - test_consume_operand %3 : !transform.any_value - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - test_consume_operand %2 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op + // expected-note @below {{handle to invalidated ops}} + %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_argument_of_parent_block %1, 0 : (!transform.any_op) -> !transform.any_value + // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}} + transform.test_consume_operand %3 : !transform.any_value + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + transform.test_consume_operand %2 : !transform.any_op + transform.yield + } } // ----- @@ -323,24 +351,28 @@ transform.sequence failures(propagate) { "test.match_anchor_2"() : () -> () }) : () -> () -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_argument_of_parent_block %1, 0 : (!transform.any_op) -> !transform.any_value - test_consume_operand %3 : !transform.any_value - test_consume_operand %2 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %1 = transform.structured.match ops{["test.match_anchor_1"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %2 = transform.structured.match ops{["test.match_anchor_2"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_argument_of_parent_block %1, 0 : (!transform.any_op) -> !transform.any_value + transform.test_consume_operand %3 : !transform.any_value + transform.test_consume_operand %2 : !transform.any_op + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.test_produce_empty_payload : !transform.any_op - // expected-note @below {{invalidated by this transform op that consumes its operand #0}} - transform.test_consume_operand %0 : !transform.any_op - // expected-error @below {{uses a handle associated with empty payload and invalidated by a previously executed transform op}} - transform.test_print_remark_at_operand %0, "remark" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_empty_payload : !transform.any_op + // expected-note @below {{invalidated by this transform op that consumes its operand #0}} + transform.test_consume_operand %0 : !transform.any_op + // expected-error @below {{uses a handle associated with empty payload and invalidated by a previously executed transform op}} + transform.test_print_remark_at_operand %0, "remark" : !transform.any_op + transform.yield + } } // ----- @@ -352,9 +384,8 @@ transform.sequence failures(propagate) { // invalidate the handle to the root module thus invalidating all other handles. // expected-note @below {{ancestor payload op}} -module { - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { // expected-note @below {{handle to invalidated ops}} // expected-note @below {{nested payload op}} %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op @@ -362,6 +393,7 @@ module { transform.test_consume_operand %arg0 : !transform.any_op // expected-error @below {{uses a handle invalidated by a previously executed transform op}} transform.test_consume_operand %0 { allow_repeated_handles } : !transform.any_op + transform.yield } } @@ -370,11 +402,13 @@ module { // Re-entering the region should not trigger the consumption error from previous // execution of the region. -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - transform.test_re_enter_region { - %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op - transform.test_consume_operand %0 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + transform.test_re_enter_region { + %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op + transform.test_consume_operand %0 : !transform.any_op + transform.yield + } transform.yield } } @@ -384,12 +418,14 @@ transform.sequence failures(propagate) { // Re-entering the region should not trigger the consumption error from previous // execution of the region. -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op - transform.test_re_enter_region %0 : !transform.any_op { - ^bb0(%arg1: !transform.any_op): - transform.test_consume_operand %arg1 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op + transform.test_re_enter_region %0 : !transform.any_op { + ^bb0(%arg1: !transform.any_op): + transform.test_consume_operand %arg1 : !transform.any_op + transform.yield + } transform.yield } } @@ -397,16 +433,17 @@ transform.sequence failures(propagate) { // ----- // Consuming the same handle repeatedly in the region should trigger an error. - -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-note @below {{payload op}} - // expected-note @below {{handle to invalidated ops}} - %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op - transform.test_re_enter_region { - // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} - // expected-note @below {{invalidated by this transform op}} - transform.test_consume_operand %0 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-note @below {{payload op}} + // expected-note @below {{handle to invalidated ops}} + %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op + transform.test_re_enter_region { + // expected-error @below {{op uses a handle invalidated by a previously executed transform op}} + // expected-note @below {{invalidated by this transform op}} + transform.test_consume_operand %0 : !transform.any_op + transform.yield + } transform.yield } } @@ -424,8 +461,8 @@ module @named_inclusion_and_consumption attributes { transform.with_named_sequen transform.yield } - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): - include @foo failures(propagate) (%arg0) : (!transform.any_op) -> () + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + transform.include @foo failures(propagate) (%arg0) : (!transform.any_op) -> () + transform.yield } } diff --git a/mlir/test/Dialect/Transform/selective-targeting.mlir b/mlir/test/Dialect/Transform/selective-targeting.mlir index 41a0d5091ec2..e88104315649 100644 --- a/mlir/test/Dialect/Transform/selective-targeting.mlir +++ b/mlir/test/Dialect/Transform/selective-targeting.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -test-transform-dialect-interpreter --split-input-file | FileCheck %s +// RUN: mlir-opt %s --transform-interpreter --split-input-file | FileCheck %s // CHECK-LABEL: func.func @matmul_tensors_1( func.func @matmul_tensors_1( @@ -52,35 +52,40 @@ func.func @matmul_tensors_3( func.return %0 : tensor<128x128xf32> } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - // Match matmul operations inside @matmul_tensors with test.attrA set. - pdl.pattern @pdl_target_attrA : benefit(1) { - %args = operands - %results = types - %attr = attribute - %0 = operation "linalg.matmul"(%args : !pdl.range) {"test.attrA" = %attr}-> (%results : !pdl.range) - // TODO: we don't want this, but it is the required terminator for pdl.pattern - rewrite %0 with "transform.dialect" - } +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root : !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + // Match matmul operations inside @matmul_tensors with test.attrA set. + pdl.pattern @pdl_target_attrA : benefit(1) { + %args = operands + %results = types + %attr = attribute + %0 = operation "linalg.matmul"(%args : !pdl.range) {"test.attrA" = %attr}-> (%results : !pdl.range) + // TODO: we don't want this, but it is the required terminator for pdl.pattern + rewrite %0 with "transform.dialect" + } - // Match matmul operations inside @matmul_tensors with test.attrC set. - pdl.pattern @pdl_target_attrC : benefit(1) { - %args = operands - %results = types - %attr = attribute - %0 = operation "linalg.matmul"(%args : !pdl.range) {"test.attrC" = %attr}-> (%results : !pdl.range) - // TODO: we don't want this, but it is the required terminator for pdl.pattern - rewrite %0 with "transform.dialect" - } + // Match matmul operations inside @matmul_tensors with test.attrC set. + pdl.pattern @pdl_target_attrC : benefit(1) { + %args = operands + %results = types + %attr = attribute + %0 = operation "linalg.matmul"(%args : !pdl.range) {"test.attrC" = %attr}-> (%results : !pdl.range) + // TODO: we don't want this, but it is the required terminator for pdl.pattern + rewrite %0 with "transform.dialect" + } - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @pdl_target_attrA in %arg1 : (!transform.any_op) -> !transform.any_op - transform.structured.tile_using_for %0 [4, 4, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op) - %1 = pdl_match @pdl_target_attrC in %arg1 : (!transform.any_op) -> !transform.any_op - %2 = get_parent_op %1 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - transform.structured.vectorize_children_and_apply_patterns %2 : (!transform.any_op) -> !transform.any_op + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @pdl_target_attrA in %arg1 : (!transform.any_op) -> !transform.any_op + transform.structured.tile_using_for %0 [4, 4, 4] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op) + %1 = pdl_match @pdl_target_attrC in %arg1 : (!transform.any_op) -> !transform.any_op + %2 = get_parent_op %1 {isolated_from_above} : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize_children_and_apply_patterns %2 : (!transform.any_op) -> !transform.any_op + } + } + transform.yield } } @@ -110,22 +115,27 @@ func.func @vectorize_none( func.return %0 : tensor<128x128xf32> } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @pdl_target : benefit(1) { - %args = operands - %results = types - %attr = attribute - %0 = operation "linalg.matmul"(%args : !pdl.range) {"test.attrA" = %attr}-> (%results : !pdl.range) - // TODO: we don't want this, but it is the required terminator for pdl.pattern - rewrite %0 with "transform.dialect" - } +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root : !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @pdl_target : benefit(1) { + %args = operands + %results = types + %attr = attribute + %0 = operation "linalg.matmul"(%args : !pdl.range) {"test.attrA" = %attr}-> (%results : !pdl.range) + // TODO: we don't want this, but it is the required terminator for pdl.pattern + rewrite %0 with "transform.dialect" + } - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @pdl_target in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @pdl_target in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op + } + } + transform.yield } } @@ -148,7 +158,9 @@ func.func @vectorize_all( return %1 : tensor<128x128xf32> } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - transform.structured.vectorize_children_and_apply_patterns %arg0 : (!transform.any_op) -> !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + transform.structured.vectorize_children_and_apply_patterns %arg0 : (!transform.any_op) -> !transform.any_op + transform.yield + } } diff --git a/mlir/test/Dialect/Transform/test-interpreter.mlir b/mlir/test/Dialect/Transform/test-interpreter.mlir index a39e6f94cb34..3bbf875ef309 100644 --- a/mlir/test/Dialect/Transform/test-interpreter.mlir +++ b/mlir/test/Dialect/Transform/test-interpreter.mlir @@ -1,29 +1,35 @@ -// RUN: mlir-opt %s --test-transform-dialect-interpreter -allow-unregistered-dialect --split-input-file --verify-diagnostics | FileCheck %s +// RUN: mlir-opt %s --transform-interpreter -allow-unregistered-dialect --split-input-file --verify-diagnostics | FileCheck %s // UNSUPPORTED: target=aarch64-pc-windows-msvc -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-remark @below {{applying transformation}} - transform.test_transform_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-remark @below {{applying transformation}} + transform.test_transform_op + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.test_produce_self_handle_or_forward_operand { foo = "bar" } : () -> !transform.any_op - // expected-remark @below {{succeeded}} - transform.test_consume_operand_of_op_kind_or_fail %0, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_self_handle_or_forward_operand { foo = "bar" } : () -> !transform.any_op + // expected-remark @below {{succeeded}} + transform.test_consume_operand_of_op_kind_or_fail %0, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.test_produce_self_handle_or_forward_operand { foo = "bar" } : () -> !transform.any_op - // expected-error @below {{expected the operand to be associated a payload op of kind transform.sequence got transform.test_produce_self_handle_or_forward_operand}} - transform.test_consume_operand_of_op_kind_or_fail %0, "transform.sequence" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_self_handle_or_forward_operand { foo = "bar" } : () -> !transform.any_op + // expected-error @below {{expected the operand to be associated a payload op of kind transform.sequence got transform.test_produce_self_handle_or_forward_operand}} + transform.test_consume_operand_of_op_kind_or_fail %0, "transform.sequence" : !transform.any_op + transform.yield + } } // ----- @@ -31,56 +37,64 @@ transform.sequence failures(propagate) { // It is okay to have multiple handles to the same payload op as long // as only one of them is consumed. The expensive checks mode is necessary // to detect double-consumption. -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.test_produce_self_handle_or_forward_operand { foo = "bar" } : () -> !transform.any_op - %1 = transform.test_copy_payload %0 : (!transform.any_op) -> !transform.any_op - // expected-remark @below {{succeeded}} - transform.test_consume_operand_of_op_kind_or_fail %0, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_self_handle_or_forward_operand { foo = "bar" } : () -> !transform.any_op + %1 = transform.test_copy_payload %0 : (!transform.any_op) -> !transform.any_op + // expected-remark @below {{succeeded}} + transform.test_consume_operand_of_op_kind_or_fail %0, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - // expected-remark @below {{applying transformation "a"}} - test_transform_op "a" - // expected-remark @below {{applying transformation "b"}} - test_transform_op "b" - // expected-remark @below {{applying transformation "c"}} - test_transform_op "c" +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + // expected-remark @below {{applying transformation "a"}} + test_transform_op "a" + // expected-remark @below {{applying transformation "b"}} + test_transform_op "b" + // expected-remark @below {{applying transformation "c"}} + test_transform_op "c" + } + // expected-remark @below {{applying transformation "d"}} + transform.test_transform_op "d" + // expected-remark @below {{applying transformation "e"}} + transform.test_transform_op "e" + transform.yield } - // expected-remark @below {{applying transformation "d"}} - test_transform_op "d" - // expected-remark @below {{applying transformation "e"}} - test_transform_op "e" } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = test_produce_self_handle_or_forward_operand : () -> !transform.any_op - sequence %0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - // expected-remark @below {{succeeded}} - test_consume_operand_of_op_kind_or_fail %arg1, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op + transform.sequence %0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + // expected-remark @below {{succeeded}} + test_consume_operand_of_op_kind_or_fail %arg1, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op + } + transform.yield } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = sequence %arg0 : !transform.any_op -> !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %1 = test_produce_self_handle_or_forward_operand : () -> !transform.any_op - yield %1 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.sequence %arg0 : !transform.any_op -> !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %1 = test_produce_self_handle_or_forward_operand : () -> !transform.any_op + yield %1 : !transform.any_op + } + // expected-remark @below {{succeeded}} + transform.test_consume_operand_of_op_kind_or_fail %0, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op + transform.yield } - // expected-remark @below {{succeeded}} - test_consume_operand_of_op_kind_or_fail %0, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op } // ----- @@ -98,19 +112,24 @@ func.func @bar() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @const : benefit(1) { - %r = pdl.types - %0 = pdl.operation "arith.constant" -> (%r : !pdl.range) - pdl.rewrite %0 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %f = pdl_match @const in %arg1 : (!transform.any_op) -> !transform.any_op - %m = get_parent_op %f {isolated_from_above} : (!transform.any_op) -> !transform.any_op - test_print_remark_at_operand %m, "parent function" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @const : benefit(1) { + %r = pdl.types + %0 = pdl.operation "arith.constant" -> (%r : !pdl.range) + pdl.rewrite %0 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %f = pdl_match @const in %arg1 : (!transform.any_op) -> !transform.any_op + %m = get_parent_op %f {isolated_from_above} : (!transform.any_op) -> !transform.any_op + test_print_remark_at_operand %m, "parent function" : !transform.any_op + } + } + transform.yield } } @@ -130,14 +149,15 @@ func.func @test_get_nth_parent() { }) : () -> () } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %f = transform.structured.match ops{["test.bar"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %parent = get_parent_op %f {nth_parent = 1, op_name = "test.foo"} : (!transform.any_op) -> !transform.any_op - test_print_remark_at_operand %parent, "1st parent" : !transform.any_op - %parent2 = get_parent_op %f {nth_parent = 2, op_name = "test.foo"} : (!transform.any_op) -> !transform.any_op - test_print_remark_at_operand %parent2, "2nd parent" : !transform.any_op - transform.yield +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %f = transform.structured.match ops{["test.bar"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %parent = transform.get_parent_op %f {nth_parent = 1, op_name = "test.foo"} : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %parent, "1st parent" : !transform.any_op + %parent2 = transform.get_parent_op %f {nth_parent = 2, op_name = "test.foo"} : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %parent2, "2nd parent" : !transform.any_op + transform.yield + } } // ----- @@ -147,32 +167,37 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @match_func : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "func.func"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - // This is necessary to run the transformation on something other than the - // top-level module, "alternatives" cannot be run on that. - %0 = pdl_match @match_func in %arg1 : (!transform.any_op) -> !transform.any_op - transform.alternatives %0 : !transform.any_op { - ^bb2(%arg2: !transform.any_op): - %1 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op - // This operation fails, which triggers the next alternative without - // reporting the error. - transform.test_consume_operand_of_op_kind_or_fail %1, "transform.sequence" : !transform.any_op - }, { - ^bb2(%arg2: !transform.any_op): - %1 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op - // expected-remark @below {{succeeded}} - transform.test_consume_operand_of_op_kind_or_fail %1, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @match_func : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "func.func"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + // This is necessary to run the transformation on something other than the + // top-level module, "alternatives" cannot be run on that. + %0 = pdl_match @match_func in %arg1 : (!transform.any_op) -> !transform.any_op + transform.alternatives %0 : !transform.any_op { + ^bb2(%arg2: !transform.any_op): + %1 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op + // This operation fails, which triggers the next alternative without + // reporting the error. + transform.test_consume_operand_of_op_kind_or_fail %1, "transform.sequence" : !transform.any_op + }, { + ^bb2(%arg2: !transform.any_op): + %1 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op + // expected-remark @below {{succeeded}} + transform.test_consume_operand_of_op_kind_or_fail %1, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op + } + } } + transform.yield } } @@ -185,26 +210,31 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @match_call : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - // expected-error @below {{all alternatives failed}} - transform.alternatives %1 : !transform.any_op { - ^bb2(%arg2: !transform.any_op): - %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op - // expected-remark @below {{applying}} - transform.test_emit_remark_and_erase_operand %2, "applying" {fail_after_erase} : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @match_call : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op + // expected-error @below {{all alternatives failed}} + transform.alternatives %1 : !transform.any_op { + ^bb2(%arg2: !transform.any_op): + %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op + // expected-remark @below {{applying}} + transform.test_emit_remark_and_erase_operand %2, "applying" {fail_after_erase} : !transform.any_op + } + } } + transform.yield } } @@ -218,35 +248,40 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @match_call : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - transform.alternatives %1 : !transform.any_op { - ^bb2(%arg2: !transform.any_op): - %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op - // expected-remark @below {{applying}} - transform.test_emit_remark_and_erase_operand %2, "applying" {fail_after_erase} : !transform.any_op - }, { - ^bb2(%arg2: !transform.any_op): - %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op - transform.test_print_remark_at_operand %2, "still here" : !transform.any_op - // This alternative succeeds. - }, { - ^bb2(%arg2: !transform.any_op): - // This alternative is never run, so we must not have a remark here. - %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op - transform.test_emit_remark_and_erase_operand %2, "should not happen" {fail_after_erase} : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @match_call : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op + transform.alternatives %1 : !transform.any_op { + ^bb2(%arg2: !transform.any_op): + %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op + // expected-remark @below {{applying}} + transform.test_emit_remark_and_erase_operand %2, "applying" {fail_after_erase} : !transform.any_op + }, { + ^bb2(%arg2: !transform.any_op): + %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %2, "still here" : !transform.any_op + // This alternative succeeds. + }, { + ^bb2(%arg2: !transform.any_op): + // This alternative is never run, so we must not have a remark here. + %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op + transform.test_emit_remark_and_erase_operand %2, "should not happen" {fail_after_erase} : !transform.any_op + } + } } + transform.yield } } @@ -259,30 +294,35 @@ func.func @erase_call() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @match_call : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - transform.alternatives %1 : !transform.any_op { - ^bb2(%arg2: !transform.any_op): - %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op - // expected-remark @below {{applying}} - transform.test_emit_remark_and_erase_operand %2, "applying" {fail_after_erase} : !transform.any_op - }, { - ^bb2(%arg2: !transform.any_op): - %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op - // expected-remark @below {{applying second time}} - transform.test_emit_remark_and_erase_operand %2, "applying second time" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @match_call : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op + transform.alternatives %1 : !transform.any_op { + ^bb2(%arg2: !transform.any_op): + %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op + // expected-remark @below {{applying}} + transform.test_emit_remark_and_erase_operand %2, "applying" {fail_after_erase} : !transform.any_op + }, { + ^bb2(%arg2: !transform.any_op): + %2 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op + // expected-remark @below {{applying second time}} + transform.test_emit_remark_and_erase_operand %2, "applying second time" : !transform.any_op + } + } } + transform.yield } } @@ -295,43 +335,48 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @match_call : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op - %2 = transform.alternatives %1 : !transform.any_op -> !transform.any_op { - ^bb2(%arg2: !transform.any_op): - %3 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op - // expected-remark @below {{applying}} - transform.test_emit_remark_and_erase_operand %3, "applying" {fail_after_erase} : !transform.any_op - %4 = transform.test_produce_self_handle_or_forward_operand %3 : (!transform.any_op) -> !transform.any_op - transform.yield %4 : !transform.any_op - }, { - ^bb2(%arg2: !transform.any_op): - %4 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op - transform.yield %4 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @match_call : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "func.call"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @match_call in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op + %2 = transform.alternatives %1 : !transform.any_op -> !transform.any_op { + ^bb2(%arg2: !transform.any_op): + %3 = transform.pdl_match @match_call in %arg2 : (!transform.any_op) -> !transform.any_op + // expected-remark @below {{applying}} + transform.test_emit_remark_and_erase_operand %3, "applying" {fail_after_erase} : !transform.any_op + %4 = transform.test_produce_self_handle_or_forward_operand %3 : (!transform.any_op) -> !transform.any_op + transform.yield %4 : !transform.any_op + }, { + ^bb2(%arg2: !transform.any_op): + %4 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op + transform.yield %4 : !transform.any_op + } + // The first alternative failed, so the returned value is taken from the + // second alternative, associated test_produce_self_handle_or_forward_operand rather + // than pdl_match. + // expected-remark @below {{succeeded}} + transform.test_consume_operand_of_op_kind_or_fail %2, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op + } } - // The first alternative failed, so the returned value is taken from the - // second alternative, associated test_produce_self_handle_or_forward_operand rather - // than pdl_match. - // expected-remark @below {{succeeded}} - transform.test_consume_operand_of_op_kind_or_fail %2, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op + transform.yield } } // ----- // expected-note @below {{scope}} -module { +module attributes {transform.with_named_sequence} { func.func @foo() { %0 = arith.constant 0 : i32 return @@ -343,8 +388,7 @@ module { return } - transform.sequence failures(propagate) { - ^bb1(%arg1: !transform.any_op): + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { // expected-error @below {{scope must not contain the transforms being applied}} transform.alternatives %arg1 : !transform.any_op { ^bb2(%arg2: !transform.any_op): @@ -355,6 +399,7 @@ module { %0 = transform.test_produce_self_handle_or_forward_operand : () -> !transform.any_op transform.test_consume_operand_of_op_kind_or_fail %0, "transform.test_produce_self_handle_or_forward_operand" : !transform.any_op } + transform.yield } } @@ -368,24 +413,29 @@ func.func @foo(%arg0: index, %arg1: index, %arg2: index) { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @match_const : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "arith.constant"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = transform.pdl_match @match_const in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = transform.get_parent_op %0 {op_name = "scf.for"} : (!transform.any_op) -> !transform.any_op - // expected-error @below {{only isolated-from-above ops can be alternative scopes}} - alternatives %1 : !transform.any_op { - ^bb2(%arg2: !transform.any_op): +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @match_const : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "arith.constant"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + + sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = transform.pdl_match @match_const in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.get_parent_op %0 {op_name = "scf.for"} : (!transform.any_op) -> !transform.any_op + // expected-error @below {{only isolated-from-above ops can be alternative scopes}} + alternatives %1 : !transform.any_op { + ^bb2(%arg2: !transform.any_op): + } + } } + transform.yield } } // ----- @@ -396,21 +446,26 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{application of transform.test_wrong_number_of_results expected to produce 3 results (actually produced 1).}} - // expected-note @below {{if you need variadic results, consider a generic `apply` instead of the specialized `applyToOne`.}} - transform.test_wrong_number_of_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{application of transform.test_wrong_number_of_results expected to produce 3 results (actually produced 1).}} + // expected-note @below {{if you need variadic results, consider a generic `apply` instead of the specialized `applyToOne`.}} + transform.test_wrong_number_of_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) + } + } + transform.yield } } @@ -423,21 +478,26 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{application of transform.test_wrong_number_of_multi_results expected to produce 1 results (actually produced 0)}} - // expected-note @below {{if you need variadic results, consider a generic `apply` instead of the specialized `applyToOne`.}} - transform.test_wrong_number_of_multi_results %0 : (!transform.any_op) -> (!transform.any_op) +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{application of transform.test_wrong_number_of_multi_results expected to produce 1 results (actually produced 0)}} + // expected-note @below {{if you need variadic results, consider a generic `apply` instead of the specialized `applyToOne`.}} + transform.test_wrong_number_of_multi_results %0 : (!transform.any_op) -> (!transform.any_op) + } + } + transform.yield } } @@ -450,20 +510,25 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // Transform matches 3 ops and produces 2 results. - %1:2 = transform.test_correct_number_of_multi_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op) +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // Transform matches 3 ops and produces 2 results. + %1:2 = transform.test_correct_number_of_multi_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + } + } + transform.yield } } @@ -474,20 +539,25 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // Transform fails to match any but still produces 2 results. - %1:2 = transform.test_correct_number_of_multi_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op) +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // Transform fails to match any but still produces 2 results. + %1:2 = transform.test_correct_number_of_multi_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + } + } + transform.yield } } @@ -500,19 +570,24 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - transform.test_mixed_null_and_non_null_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op) +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_mixed_null_and_non_null_results %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + } + } + transform.yield } } @@ -530,27 +605,32 @@ func.func @foo(%arg0: index) { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @addi : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "arith.addi"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - pdl.pattern @subi : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "arith.subi"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @addi in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = pdl_match @subi in %arg1 : (!transform.any_op) -> !transform.any_op - %2 = merge_handles %0, %1 : !transform.any_op - test_print_remark_at_operand %2, "matched" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @addi : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "arith.addi"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + pdl.pattern @subi : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "arith.subi"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @addi in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = pdl_match @subi in %arg1 : (!transform.any_op) -> !transform.any_op + %2 = merge_handles %0, %1 : !transform.any_op + test_print_remark_at_operand %2, "matched" : !transform.any_op + } + } + transform.yield } } @@ -561,23 +641,28 @@ func.func @foo(%arg0: index) { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @addi : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "arith.addi"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @addi in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = pdl_match @addi in %arg1 : (!transform.any_op) -> !transform.any_op - %2 = merge_handles deduplicate %0, %1 : !transform.any_op - %3 = num_associations %2 : (!transform.any_op) -> !transform.param - // expected-remark @below {{1}} - test_print_param %3 : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @addi : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "arith.addi"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @addi in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = pdl_match @addi in %arg1 : (!transform.any_op) -> !transform.any_op + %2 = merge_handles deduplicate %0, %1 : !transform.any_op + %3 = num_associations %2 : (!transform.any_op) -> !transform.param + // expected-remark @below {{1}} + test_print_param %3 : !transform.param + } + } + transform.yield } } @@ -590,20 +675,25 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{failed to apply}} - transform.test_mixed_success_and_silenceable %0 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{failed to apply}} + transform.test_mixed_success_and_silenceable %0 : !transform.any_op + } + } + transform.yield } } @@ -614,21 +704,26 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(suppress) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // Not expecting error here because we are suppressing it. - // expected-remark @below {{foo}} - test_emit_remark_and_erase_operand %0, "foo" {fail_after_erase} : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(suppress) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // Not expecting error here because we are suppressing it. + // expected-remark @below {{foo}} + test_emit_remark_and_erase_operand %0, "foo" {fail_after_erase} : !transform.any_op + } + } + transform.yield } } @@ -639,52 +734,61 @@ func.func @foo() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{silenceable error}} - // expected-remark @below {{foo}} - test_emit_remark_and_erase_operand %0, "foo" {fail_after_erase} : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "op"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{silenceable error}} + // expected-remark @below {{foo}} + test_emit_remark_and_erase_operand %0, "foo" {fail_after_erase} : !transform.any_op + } + } + transform.yield } } // ----- -module { +module attributes {transform.with_named_sequence} { func.func private @foo() func.func private @bar() - transform.with_pdl_patterns { - ^bb0(%arg0: !transform.any_op): - pdl.pattern @func : benefit(1) { - %0 = pdl.operands - %1 = pdl.types - %2 = pdl.operation "func.func"(%0 : !pdl.range) -> (%1 : !pdl.range) - pdl.rewrite %2 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @func in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = replicate num(%0) %arg1 : !transform.any_op, !transform.any_op - %p = num_associations %1 : (!transform.any_op) -> !transform.param - // expected-remark @below {{2}} - test_print_param %p : !transform.param - %2 = replicate num(%0) %1 : !transform.any_op, !transform.any_op - %p2 = num_associations %2 : (!transform.any_op) -> !transform.param - // expected-remark @below {{4}} - test_print_param %p2 : !transform.param + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + + ^bb0(%arg0: !transform.any_op): + pdl.pattern @func : benefit(1) { + %0 = pdl.operands + %1 = pdl.types + %2 = pdl.operation "func.func"(%0 : !pdl.range) -> (%1 : !pdl.range) + pdl.rewrite %2 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @func in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = replicate num(%0) %arg1 : !transform.any_op, !transform.any_op + %p = num_associations %1 : (!transform.any_op) -> !transform.param + // expected-remark @below {{2}} + test_print_param %p : !transform.param + %2 = replicate num(%0) %1 : !transform.any_op, !transform.any_op + %p2 = num_associations %2 : (!transform.any_op) -> !transform.param + // expected-remark @below {{4}} + test_print_param %p2 : !transform.param + } } + transform.yield } } @@ -698,24 +802,29 @@ func.func @bar() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @const : benefit(1) { - %r = pdl.types - %0 = pdl.operation "arith.constant" -> (%r : !pdl.range) - pdl.rewrite %0 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %f = pdl_match @const in %arg1 : (!transform.any_op) -> !transform.any_op - transform.foreach %f : !transform.any_op { - ^bb2(%arg2: !transform.any_op): - %p = transform.num_associations %arg2 : (!transform.any_op) -> !transform.param - // expected-remark @below {{1}} - transform.test_print_param %p : !transform.param - transform.test_print_remark_at_operand %arg2, "transform applied" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @const : benefit(1) { + %r = pdl.types + %0 = pdl.operation "arith.constant" -> (%r : !pdl.range) + pdl.rewrite %0 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %f = pdl_match @const in %arg1 : (!transform.any_op) -> !transform.any_op + transform.foreach %f : !transform.any_op { + ^bb2(%arg2: !transform.any_op): + %p = transform.num_associations %arg2 : (!transform.any_op) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p : !transform.param + transform.test_print_remark_at_operand %arg2, "transform applied" : !transform.any_op + } + } } + transform.yield } } @@ -731,13 +840,15 @@ func.func @consume_in_foreach() { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %f = transform.structured.match ops{["arith.constant"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.foreach %f : !transform.any_op { - ^bb2(%arg2: !transform.any_op): - // expected-remark @below {{erasing}} - transform.test_emit_remark_and_erase_operand %arg2, "erasing" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %f = transform.structured.match ops{["arith.constant"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.foreach %f : !transform.any_op { + ^bb2(%arg2: !transform.any_op): + // expected-remark @below {{erasing}} + transform.test_emit_remark_and_erase_operand %arg2, "erasing" : !transform.any_op + } + transform.yield } } @@ -761,33 +872,38 @@ func.func @bar() { return } -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @const : benefit(1) { - %r = pdl.types - %0 = pdl.operation "arith.constant" -> (%r : !pdl.range) - pdl.rewrite %0 with "transform.dialect" - } - - pdl.pattern @execute_region : benefit(1) { - %r = pdl.types - %0 = pdl.operation "scf.execute_region" -> (%r : !pdl.range) - pdl.rewrite %0 with "transform.dialect" - } - - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %f = pdl_match @execute_region in %arg1 : (!transform.any_op) -> !transform.any_op - %results = transform.foreach %f : !transform.any_op -> !transform.any_op { - ^bb2(%arg2: !transform.any_op): - %g = transform.pdl_match @const in %arg2 : (!transform.any_op) -> !transform.any_op - transform.yield %g : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @const : benefit(1) { + %r = pdl.types + %0 = pdl.operation "arith.constant" -> (%r : !pdl.range) + pdl.rewrite %0 with "transform.dialect" + } + + pdl.pattern @execute_region : benefit(1) { + %r = pdl.types + %0 = pdl.operation "scf.execute_region" -> (%r : !pdl.range) + pdl.rewrite %0 with "transform.dialect" + } + + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %f = pdl_match @execute_region in %arg1 : (!transform.any_op) -> !transform.any_op + %results = transform.foreach %f : !transform.any_op -> !transform.any_op { + ^bb2(%arg2: !transform.any_op): + %g = transform.pdl_match @const in %arg2 : (!transform.any_op) -> !transform.any_op + transform.yield %g : !transform.any_op + } + + %p = transform.num_associations %results : (!transform.any_op) -> !transform.param + // expected-remark @below {{3}} + transform.test_print_param %p : !transform.param + transform.test_print_remark_at_operand %results, "transform applied" : !transform.any_op + } } - - %p = transform.num_associations %results : (!transform.any_op) -> !transform.param - // expected-remark @below {{3}} - transform.test_print_param %p : !transform.param - transform.test_print_remark_at_operand %results, "transform applied" : !transform.any_op + transform.yield } } @@ -800,11 +916,13 @@ func.func @get_parent_for_op_no_loop(%arg0: index, %arg1: index) { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %muli = get_producer_of_operand %addi[0] : (!transform.any_op) -> !transform.any_op - transform.test_print_remark_at_operand %muli, "found muli" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %muli = transform.get_producer_of_operand %addi[0] : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %muli, "found muli" : !transform.any_op + transform.yield + } } // ----- @@ -815,12 +933,13 @@ func.func @get_parent_for_op_no_loop(%arg0: index, %arg1: index) { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{could not find a producer for operand number: 0 of}} - %bbarg = get_producer_of_operand %muli[0] : (!transform.any_op) -> !transform.any_op - +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{could not find a producer for operand number: 0 of}} + %bbarg = transform.get_producer_of_operand %muli[0] : (!transform.any_op) -> !transform.any_op + transform.yield + } } // ----- @@ -832,11 +951,13 @@ func.func @get_consumer(%arg0: index, %arg1: index) { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %addi = get_consumers_of_result %muli[0] : (!transform.any_op) -> !transform.any_op - transform.test_print_remark_at_operand %addi, "found addi" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %addi = transform.get_consumers_of_result %muli[0] : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %addi, "found addi" : !transform.any_op + transform.yield + } } // ----- @@ -847,12 +968,13 @@ func.func @get_consumer_fail_1(%arg0: index, %arg1: index) { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{handle must be mapped to exactly one payload op}} - %bbarg = get_consumers_of_result %muli[0] : (!transform.any_op) -> !transform.any_op - +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{handle must be mapped to exactly one payload op}} + %bbarg = transform.get_consumers_of_result %muli[0] : (!transform.any_op) -> !transform.any_op + transform.yield + } } // ----- @@ -862,12 +984,13 @@ func.func @get_consumer_fail_2(%arg0: index, %arg1: index) { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{result number overflow}} - %bbarg = get_consumers_of_result %muli[1] : (!transform.any_op) -> !transform.any_op - +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %muli = transform.structured.match ops{["arith.muli"]} in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{result number overflow}} + %bbarg = transform.get_consumers_of_result %muli[1] : (!transform.any_op) -> !transform.any_op + transform.yield + } } // ----- @@ -878,16 +1001,18 @@ func.func @split_handle(%a: index, %b: index, %c: index) { return } -transform.sequence failures(propagate) { -^bb1(%fun: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op - %h:2 = split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op) - %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param - // expected-remark @below {{1}} - transform.test_print_param %p : !transform.param - %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op - // expected-error @below {{expected to contain 3 payload ops but it contains 2 payload ops}} - %h_2:3 = split_handle %muli_2 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%fun: !transform.any_op) { + %muli = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op + %h:2 = transform.split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p : !transform.param + %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op + // expected-error @below {{expected to contain 3 payload ops but it contains 2 payload ops}} + %h_2:3 = transform.split_handle %muli_2 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) + transform.yield + } } // ----- @@ -898,19 +1023,24 @@ func.func @split_handle(%a: index, %b: index, %c: index) { return } -transform.sequence failures(suppress) { -^bb1(%fun: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op - %h:2 = split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op) - %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param - // expected-remark @below {{1}} - transform.test_print_param %p : !transform.param - %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op - // Silenceable failure and all handles are now empty. - %h_2:3 = split_handle %muli_2 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) - %p2 = transform.num_associations %h_2#0 : (!transform.any_op) -> !transform.param - // expected-remark @below {{0}} - transform.test_print_param %p2 : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.sequence %root : !transform.any_op failures(suppress) { + ^bb1(%fun: !transform.any_op): + %muli = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op + %h:2 = split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p : !transform.param + %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op + // Silenceable failure and all handles are now empty. + %h_2:3 = split_handle %muli_2 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) + %p2 = transform.num_associations %h_2#0 : (!transform.any_op) -> !transform.param + // expected-remark @below {{0}} + transform.test_print_param %p2 : !transform.param + } + transform.yield + } } // ----- @@ -921,20 +1051,22 @@ func.func @split_handle(%a: index, %b: index, %c: index) { return } -transform.sequence failures(propagate) { -^bb1(%fun: !transform.any_op): - %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op - // No error, last result handle is empty. - %h:3 = split_handle %muli_2 {fail_on_payload_too_small = false} : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) - %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param - // expected-remark @below {{1}} - transform.test_print_param %p : !transform.param - %p2 = transform.num_associations %h#1 : (!transform.any_op) -> !transform.param - // expected-remark @below {{1}} - transform.test_print_param %p2 : !transform.param - %p3 = transform.num_associations %h#2 : (!transform.any_op) -> !transform.param - // expected-remark @below {{0}} - transform.test_print_param %p3 : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%fun: !transform.any_op) { + %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op + // No error, last result handle is empty. + %h:3 = transform.split_handle %muli_2 {fail_on_payload_too_small = false} : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) + %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p : !transform.param + %p2 = transform.num_associations %h#1 : (!transform.any_op) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p2 : !transform.param + %p3 = transform.num_associations %h#2 : (!transform.any_op) -> !transform.param + // expected-remark @below {{0}} + transform.test_print_param %p3 : !transform.param + transform.yield + } } // ----- @@ -947,16 +1079,18 @@ func.func @split_handle(%a: index, %b: index, %c: index) { return } -transform.sequence failures(propagate) { -^bb1(%fun: !transform.any_op): - %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op - %h:2 = split_handle %muli_2 {overflow_result = 0} : (!transform.any_op) -> (!transform.any_op, !transform.any_op) - %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param - // expected-remark @below {{3}} - transform.test_print_param %p : !transform.param - %p2 = transform.num_associations %h#1 : (!transform.any_op) -> !transform.param - // expected-remark @below {{1}} - transform.test_print_param %p2 : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%fun: !transform.any_op) { + %muli_2 = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op + %h:2 = transform.split_handle %muli_2 {overflow_result = 0} : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + %p = transform.num_associations %h#0 : (!transform.any_op) -> !transform.param + // expected-remark @below {{3}} + transform.test_print_param %p : !transform.param + %p2 = transform.num_associations %h#1 : (!transform.any_op) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p2 : !transform.param + transform.yield + } } // ----- @@ -964,18 +1098,23 @@ transform.sequence failures(propagate) { "test.some_op"() : () -> () "other_dialect.other_op"() : () -> () -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operation "test.some_op" - pdl.rewrite %0 with "transform.dialect" - } - - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - %2 = transform.cast %0 : !transform.any_op to !transform.test_dialect_op - transform.cast %2 : !transform.test_dialect_op to !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operation "test.some_op" + pdl.rewrite %0 with "transform.dialect" + } + + sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + %2 = transform.cast %0 : !transform.any_op to !transform.test_dialect_op + transform.cast %2 : !transform.test_dialect_op to !transform.any_op + } + } + transform.yield } } @@ -984,19 +1123,24 @@ transform.with_pdl_patterns { "test.some_op"() : () -> () "other_dialect.other_op"() : () -> () -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @other : benefit(1) { - %0 = pdl.operation "other_dialect.other_op" - pdl.rewrite %0 with "transform.dialect" - } - - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @other in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{expected the payload operation to belong to the 'test' dialect}} - %2 = transform.cast %0 : !transform.any_op to !transform.test_dialect_op - transform.cast %2 : !transform.test_dialect_op to !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @other : benefit(1) { + %0 = pdl.operation "other_dialect.other_op" + pdl.rewrite %0 with "transform.dialect" + } + + sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @other in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{expected the payload operation to belong to the 'test' dialect}} + %2 = transform.cast %0 : !transform.any_op to !transform.test_dialect_op + transform.cast %2 : !transform.test_dialect_op to !transform.any_op + } + } + transform.yield } } @@ -1005,18 +1149,23 @@ transform.with_pdl_patterns { "test.some_op"() : () -> () "other_dialect.other_op"() : () -> () -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @some : benefit(1) { - %0 = pdl.operation "test.some_op" - pdl.rewrite %0 with "transform.dialect" - } - - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - %2 = transform.cast %0 : !transform.any_op to !transform.op<"test.some_op"> - transform.cast %2 : !transform.op<"test.some_op"> to !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @some : benefit(1) { + %0 = pdl.operation "test.some_op" + pdl.rewrite %0 with "transform.dialect" + } + + sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + %2 = transform.cast %0 : !transform.any_op to !transform.op<"test.some_op"> + transform.cast %2 : !transform.op<"test.some_op"> to !transform.any_op + } + } + transform.yield } } @@ -1026,42 +1175,52 @@ transform.with_pdl_patterns { // expected-note @below {{payload operation}} "other_dialect.other_op"() : () -> () -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - pdl.pattern @other : benefit(1) { - %0 = pdl.operation "other_dialect.other_op" - pdl.rewrite %0 with "transform.dialect" - } - - sequence %arg0 : !transform.any_op failures(propagate) { - ^bb1(%arg1: !transform.any_op): - %0 = pdl_match @other in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{incompatible payload operation name}} - %2 = transform.cast %0 : !transform.any_op to !transform.op<"test.some_op"> - transform.cast %2 : !transform.op<"test.some_op"> to !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + pdl.pattern @other : benefit(1) { + %0 = pdl.operation "other_dialect.other_op" + pdl.rewrite %0 with "transform.dialect" + } + + sequence %arg0 : !transform.any_op failures(propagate) { + ^bb1(%arg1: !transform.any_op): + %0 = pdl_match @other in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{incompatible payload operation name}} + %2 = transform.cast %0 : !transform.any_op to !transform.op<"test.some_op"> + transform.cast %2 : !transform.op<"test.some_op"> to !transform.any_op + } + } + transform.yield } } // ----- -transform.with_pdl_patterns { -^bb0(%arg0: !transform.any_op): - transform.sequence %arg0 : !transform.any_op failures(propagate) { - ^bb0(%arg1: !transform.any_op): - %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op - // here, the handles nested under are {%arg0, %arg1, %0} - // expected-remark @below {{3 handles nested under}} - transform.test_report_number_of_tracked_handles_nested_under %arg1 : !transform.any_op - // expected-remark @below {{erased}} - transform.test_emit_remark_and_erase_operand %0, "erased" : !transform.any_op - // here, the handles nested under are only {%arg0, %arg1} - // expected-remark @below {{2 handles nested under}} - transform.test_report_number_of_tracked_handles_nested_under %arg1 : !transform.any_op - } +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.with_pdl_patterns %root : !transform.any_op { + ^bb0(%arg0: !transform.any_op): + transform.sequence %arg0 : !transform.any_op failures(propagate) { + ^bb0(%arg1: !transform.any_op): + %0 = pdl_match @some in %arg1 : (!transform.any_op) -> !transform.any_op + // here, the handles nested under are {%root, %arg0, %arg1, %0} + // expected-remark @below {{4 handles nested under}} + transform.test_report_number_of_tracked_handles_nested_under %arg1 : !transform.any_op + // expected-remark @below {{erased}} + transform.test_emit_remark_and_erase_operand %0, "erased" : !transform.any_op + // here, the handles nested under are only {%root, %arg0, %arg1} + // expected-remark @below {{3 handles nested under}} + transform.test_report_number_of_tracked_handles_nested_under %arg1 : !transform.any_op + } - pdl.pattern @some : benefit(1) { - %0 = pdl.operation "test.some_op" - pdl.rewrite %0 with "transform.dialect" + pdl.pattern @some : benefit(1) { + %0 = pdl.operation "test.some_op" + pdl.rewrite %0 with "transform.dialect" + } + } + transform.yield } } @@ -1075,66 +1234,84 @@ func.func @split_handle(%a: index, %b: index, %c: index) { return } -transform.sequence -> !transform.any_op failures(propagate) { -^bb1(%fun: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op - // expected-error @below {{expected to contain 3 payload ops but it contains 2 payload ops}} - %h_2:3 = split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) - /// Test that yield does not crash in the presence of silenceable error in - /// propagate mode. - yield %fun : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.sequence %root : !transform.any_op -> !transform.any_op failures(propagate) { + ^bb1(%fun: !transform.any_op): + %muli = transform.structured.match ops{["arith.muli"]} in %fun : (!transform.any_op) -> !transform.any_op + // expected-error @below {{expected to contain 3 payload ops but it contains 2 payload ops}} + %h_2:3 = split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) + /// Test that yield does not crash in the presence of silenceable error in + /// propagate mode. + yield %fun : !transform.any_op + } + transform.yield + } } // ----- -transform.sequence -> !transform.any_op failures(suppress) { -^bb0(%arg0: !transform.any_op): - %muli = transform.structured.match ops{["arith.muli"]} in %arg0 : (!transform.any_op) -> !transform.any_op - // Edge case propagating empty handles in splitting. - %0:3 = split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) - // Test does not crash when accessing the empty handle. - yield %0#0 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.sequence %root : !transform.any_op -> !transform.any_op failures(suppress) { + ^bb0(%arg0: !transform.any_op): + %muli = transform.structured.match ops{["arith.muli"]} in %arg0 : (!transform.any_op) -> !transform.any_op + // Edge case propagating empty handles in splitting. + %0:3 = split_handle %muli : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op) + // Test does not crash when accessing the empty handle. + yield %0#0 : !transform.any_op + } + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.test_produce_param (0 : i32) : !transform.test_dialect_param - // expected-remark @below {{0 : i32}} - transform.test_print_param %0 : !transform.test_dialect_param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_param (0 : i32) : !transform.test_dialect_param + // expected-remark @below {{0 : i32}} + transform.test_print_param %0 : !transform.test_dialect_param + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-error @below {{expected the type of the parameter attribute ('i32') to match the parameter type ('i64')}} - transform.test_produce_param (0 : i32) : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-error @below {{expected the type of the parameter attribute ('i32') to match the parameter type ('i64')}} + transform.test_produce_param (0 : i32) : !transform.param + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.test_add_to_param 40 - %1 = transform.test_add_to_param %0, 2 - // expected-remark @below {{42 : i32}} - transform.test_print_param %1 : !transform.test_dialect_param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_add_to_param 40 + %1 = transform.test_add_to_param %0, 2 + // expected-remark @below {{42 : i32}} + transform.test_print_param %1 : !transform.test_dialect_param + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %1 = transform.test_produce_param_with_number_of_test_ops %0 : !transform.any_op - // expected-remark @below {{1 : i32, 3 : i32}} - transform.test_print_param %1 : !transform.test_dialect_param - %2 = transform.test_add_to_param %1, 100 - // expected-remark @below {{101 : i32, 103 : i32}} - transform.test_print_param %2 : !transform.test_dialect_param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %1 = transform.test_produce_param_with_number_of_test_ops %0 : !transform.any_op + // expected-remark @below {{1 : i32, 3 : i32}} + transform.test_print_param %1 : !transform.test_dialect_param + %2 = transform.test_add_to_param %1, 100 + // expected-remark @below {{101 : i32, 103 : i32}} + transform.test_print_param %2 : !transform.test_dialect_param + transform.yield + } } func.func private @one_test_op(%arg0: i32) { @@ -1152,13 +1329,13 @@ func.func private @three_test_ops(%arg0: i32) { // ----- // expected-note @below {{when applied to this op}} -module { - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { // expected-error @below {{expected to produce an Operation * for result #0}} transform.test_produce_transform_param_or_forward_operand %arg0 { first_result_is_param } : (!transform.any_op) -> (!transform.any_op, !transform.param) + transform.yield } } @@ -1166,92 +1343,88 @@ module { // Should not fail. -module { - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { transform.test_produce_transform_param_or_forward_operand %arg0 { first_result_is_null } : (!transform.any_op) -> (!transform.any_op, !transform.param) + transform.yield } } // ----- // expected-note @below {{when applied to this op}} -module { - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { // expected-error @below {{expected to produce an Attribute for result #1}} transform.test_produce_transform_param_or_forward_operand %arg0 { second_result_is_handle } : (!transform.any_op) -> (!transform.any_op, !transform.param) + transform.yield } } // ----- // expected-note @below {{when applied to this op}} -module { - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { // expected-error @below {{expected to produce a Value for result #0}} transform.test_produce_transform_param_or_forward_operand %arg0 { second_result_is_handle } : (!transform.any_op) -> (!transform.any_value, !transform.param) + transform.yield } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-error @below {{attempting to assign a null payload op to this transform value}} - %0 = transform.test_produce_null_payload : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-error @below {{attempting to assign a null payload op to this transform value}} + %0 = transform.test_produce_null_payload : !transform.any_op + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-error @below {{attempting to assign a null parameter to this transform value}} - %0 = transform.test_produce_null_param : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-error @below {{attempting to assign a null parameter to this transform value}} + %0 = transform.test_produce_null_param : !transform.param + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-error @below {{attempting to assign a null payload value to this transform handle}} - %0 = transform.test_produce_null_value : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-error @below {{attempting to assign a null payload value to this transform handle}} + %0 = transform.test_produce_null_value : !transform.any_value + transform.yield + } } // ----- -// expected-error @below {{could not find a nested top-level transform op}} -// expected-note @below {{use the 'transform-file-name' option to provide transform as external file}} +// expected-error @below {{could not find a nested named sequence with name: __transform_main}} +// expected-error @below {{could not find transform entry point: __transform_main in either payload or transform module}} module { } // ----- -// expected-note @below {{previous top-level transform op}} -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): -} - -// expected-error @below {{more than one top-level transform op}} -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): -} - -// ----- - -transform.sequence failures(propagate) { -// expected-remark @below {{value handle}} -// expected-note @below {{value handle points to a block argument #0 in block #0 in region #0}} -^bb1(%arg0: !transform.any_op): - %0 = test_produce_value_handle_to_self_operand %arg0 : (!transform.any_op) -> !transform.any_value - test_print_remark_at_operand_value %0, "value handle" : !transform.any_value +module attributes {transform.with_named_sequence} { + // expected-remark @below {{value handle}} + // expected-note @below {{value handle points to a block argument #0 in block #0 in region #0}} + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.test_produce_value_handle_to_self_operand %arg0 : (!transform.any_op) -> !transform.any_value + transform.test_print_remark_at_operand_value %0, "value handle" : !transform.any_value + transform.yield + } } // ----- @@ -1263,11 +1436,13 @@ transform.sequence failures(propagate) { // expected-note @below {{value handle points to an op result #1}} %1:3 = "test.get_three_results"() : () -> (i32, i32, f32) -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %2 = transform.structured.match ops{["test.get_two_results", "test.get_three_results"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_result %2, 1 : (!transform.any_op) -> !transform.any_value - test_print_remark_at_operand_value %3, "result handle" : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %2 = transform.structured.match ops{["test.get_two_results", "test.get_three_results"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_result %2, 1 : (!transform.any_op) -> !transform.any_value + transform.test_print_remark_at_operand_value %3, "result handle" : !transform.any_value + transform.yield + } } // ----- @@ -1285,21 +1460,25 @@ transform.sequence failures(propagate) { "test.regon_terminator"() : () -> () }) : () -> () -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %3 = test_produce_value_handle_to_argument_of_parent_block %2, 2 : (!transform.any_op) -> !transform.any_value - test_print_remark_at_operand_value %3, "block argument handle" : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %2 = transform.structured.match ops{["test.match_anchor"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %3 = transform.test_produce_value_handle_to_argument_of_parent_block %2, 2 : (!transform.any_op) -> !transform.any_value + transform.test_print_remark_at_operand_value %3, "block argument handle" : !transform.any_value + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-note @below {{value defined here with type '!transform.test_dialect_param'}} - %0 = test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op - // expected-error @below {{unexpectedly consumed a value that is not a handle as operand #0}} - test_consume_operand %0 : !transform.test_dialect_param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-note @below {{value defined here with type '!transform.test_dialect_param'}} + %0 = transform.test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op + // expected-error @below {{unexpectedly consumed a value that is not a handle as operand #0}} + transform.test_consume_operand %0 : !transform.test_dialect_param + transform.yield + } } // ----- @@ -1311,11 +1490,13 @@ func.func @get_result_of_op(%arg0: index, %arg1: index) -> index { return %r : index } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %result = transform.get_result %addi[0] : (!transform.any_op) -> !transform.any_value - transform.test_print_remark_at_operand_value %result, "addi result" : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %result = transform.get_result %addi[0] : (!transform.any_op) -> !transform.any_value + transform.test_print_remark_at_operand_value %result, "addi result" : !transform.any_value + transform.yield + } } // ----- @@ -1326,12 +1507,14 @@ func.func @get_out_of_bounds_result_of_op(%arg0: index, %arg1: index) -> index { return %r : index } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-error @below {{targeted op does not have enough results}} - %result = transform.get_result %addi[1] : (!transform.any_op) -> !transform.any_value - transform.test_print_remark_at_operand_value %result, "addi result" : !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-error @below {{targeted op does not have enough results}} + %result = transform.get_result %addi[1] : (!transform.any_op) -> !transform.any_value + transform.test_print_remark_at_operand_value %result, "addi result" : !transform.any_value + transform.yield + } } // ----- @@ -1342,12 +1525,14 @@ func.func @get_result_of_op(%arg0: index, %arg1: index) -> index { return %r : index } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %result = transform.get_result %addi[0] : (!transform.any_op) -> !transform.any_value - %op = transform.get_defining_op %result : (!transform.any_value) -> !transform.any_op - transform.test_print_remark_at_operand %op, "matched" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %result = transform.get_result %addi[0] : (!transform.any_op) -> !transform.any_value + %op = transform.get_defining_op %result : (!transform.any_value) -> !transform.any_op + transform.test_print_remark_at_operand %op, "matched" : !transform.any_op + transform.yield + } } // ----- @@ -1358,13 +1543,15 @@ func.func @get_result_of_op_bbarg(%arg0: index, %arg1: index) -> index { return %r : index } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %bbarg = test_produce_value_handle_to_argument_of_parent_block %addi, 0 : (!transform.any_op) -> !transform.any_value - // expected-error @below {{cannot get defining op of block argument}} - %op = transform.get_defining_op %bbarg : (!transform.any_value) -> !transform.any_op - transform.test_print_remark_at_operand %op, "matched" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %addi = transform.structured.match ops{["arith.addi"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %bbarg = transform.test_produce_value_handle_to_argument_of_parent_block %addi, 0 : (!transform.any_op) -> !transform.any_value + // expected-error @below {{cannot get defining op of block argument}} + %op = transform.get_defining_op %bbarg : (!transform.any_value) -> !transform.any_op + transform.test_print_remark_at_operand %op, "matched" : !transform.any_op + transform.yield + } } // ----- @@ -1377,9 +1564,9 @@ module @named_inclusion attributes { transform.with_named_sequence } { transform.yield } - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): - include @foo failures(propagate) (%arg0) : (!transform.any_op) -> () + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + transform.include @foo failures(propagate) (%arg0) : (!transform.any_op) -> () + transform.yield } } @@ -1400,9 +1587,9 @@ module @named_inclusion_in_named attributes { transform.with_named_sequence } { transform.yield } - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { transform.include @bar failures(suppress) (%arg0) : (!transform.any_op) -> () + transform.yield } } @@ -1418,12 +1605,12 @@ module @named_operands attributes { transform.with_named_sequence } { transform.yield } - transform.sequence failures(propagate) { // expected-remark @below {{value}} // expected-note @below {{value handle points to a block argument #0 in block #0 in region #0}} - ^bb0(%arg0: !transform.any_op): + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { %0 = transform.test_produce_value_handle_to_self_operand %arg0 : (!transform.any_op) -> !transform.any_value - include @foo failures(propagate) (%arg0, %0) : (!transform.any_op, !transform.any_value) -> () + transform.include @foo failures(propagate) (%arg0, %0) : (!transform.any_op, !transform.any_value) -> () + transform.yield } } @@ -1439,11 +1626,11 @@ module @named_return attributes { transform.with_named_sequence } { transform.yield %arg0, %0 : !transform.any_op, !transform.any_value } - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): - %0:2 = include @foo failures(propagate) (%arg0) : (!transform.any_op) -> (!transform.any_op, !transform.any_value) + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0:2 = transform.include @foo failures(propagate) (%arg0) : (!transform.any_op) -> (!transform.any_op, !transform.any_value) transform.test_print_remark_at_operand %0#0, "operation" : !transform.any_op transform.test_print_remark_at_operand_value %0#1, "value" : !transform.any_value + transform.yield } } @@ -1469,8 +1656,7 @@ module attributes { transform.with_named_sequence } { transform.yield } - transform.sequence failures(propagate) { - ^bb0(%root: !transform.any_op): + transform.named_sequence @__transform_main(%root: !transform.any_op) { transform.foreach_match in %root @match1 -> @action1, @match2 -> @action2 @@ -1493,11 +1679,11 @@ module attributes { transform.with_named_sequence } { transform.named_sequence @match(!transform.any_op {transform.readonly}) transform.named_sequence @action() - transform.sequence failures(propagate) { - ^bb0(%root: !transform.any_op): + transform.named_sequence @__transform_main(%root: !transform.any_op) { // expected-error @below {{unresolved external symbol @match}} transform.foreach_match in %root @match -> @action : (!transform.any_op) -> !transform.any_op + transform.yield } } @@ -1509,11 +1695,11 @@ module attributes { transform.with_named_sequence } { } transform.named_sequence @action() - transform.sequence failures(propagate) { - ^bb0(%root: !transform.any_op): + transform.named_sequence @__transform_main(%root: !transform.any_op) { // expected-error @below {{unresolved external symbol @action}} transform.foreach_match in %root @match -> @action : (!transform.any_op) -> !transform.any_op + transform.yield } } @@ -1529,17 +1715,17 @@ module attributes { transform.with_named_sequence } { transform.yield } - transform.sequence failures(propagate) { - ^bb0(%root: !transform.any_op): + transform.named_sequence @__transform_main(%root: !transform.any_op) { transform.foreach_match in %root @match -> @action : (!transform.any_op) -> !transform.any_op + transform.yield } } // ----- module attributes { transform.with_named_sequence } { - transform.named_sequence @match_func(%arg0: !transform.any_op {transform.readonly}) + transform.named_sequence @match_func(%arg0: !transform.any_op {transform.readonly}) -> !transform.any_op { transform.match.operation_name %arg0 ["func.func"] : !transform.any_op transform.yield %arg0 : !transform.any_op @@ -1550,8 +1736,7 @@ module attributes { transform.with_named_sequence } { transform.yield } - transform.sequence failures(propagate) { - ^bb(%arg0: !transform.any_op): + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { transform.foreach_match in %arg0 @match_func -> @print_func : (!transform.any_op) -> !transform.any_op transform.yield } @@ -1570,7 +1755,7 @@ module attributes { transform.with_named_sequence } { // ----- module attributes { transform.with_named_sequence } { - transform.named_sequence @eq_1(%arg0: !transform.any_op {transform.readonly}) + transform.named_sequence @eq_1(%arg0: !transform.any_op {transform.readonly}) -> !transform.any_op { transform.match.operation_name %arg0 ["func.func"] : !transform.any_op %0 = transform.test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op @@ -1580,7 +1765,7 @@ module attributes { transform.with_named_sequence } { transform.yield %arg0 : !transform.any_op } - transform.named_sequence @ne_0(%arg0: !transform.any_op {transform.readonly}) + transform.named_sequence @ne_0(%arg0: !transform.any_op {transform.readonly}) -> !transform.any_op { transform.match.operation_name %arg0 ["func.func"] : !transform.any_op %0 = transform.test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op @@ -1590,7 +1775,7 @@ module attributes { transform.with_named_sequence } { transform.yield %arg0 : !transform.any_op } - transform.named_sequence @gt_m1(%arg0: !transform.any_op {transform.readonly}) + transform.named_sequence @gt_m1(%arg0: !transform.any_op {transform.readonly}) -> !transform.any_op { transform.match.operation_name %arg0 ["func.func"] : !transform.any_op %0 = transform.test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op @@ -1600,7 +1785,7 @@ module attributes { transform.with_named_sequence } { transform.yield %arg0 : !transform.any_op } - transform.named_sequence @ge_1(%arg0: !transform.any_op {transform.readonly}) + transform.named_sequence @ge_1(%arg0: !transform.any_op {transform.readonly}) -> !transform.any_op { transform.match.operation_name %arg0 ["func.func"] : !transform.any_op %0 = transform.test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op @@ -1610,7 +1795,7 @@ module attributes { transform.with_named_sequence } { transform.yield %arg0 : !transform.any_op } - transform.named_sequence @lt_1(%arg0: !transform.any_op {transform.readonly}) + transform.named_sequence @lt_1(%arg0: !transform.any_op {transform.readonly}) -> !transform.any_op { transform.match.operation_name %arg0 ["func.func"] : !transform.any_op %0 = transform.test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op @@ -1620,7 +1805,7 @@ module attributes { transform.with_named_sequence } { transform.yield %arg0 : !transform.any_op } - transform.named_sequence @le_1(%arg0: !transform.any_op {transform.readonly}) + transform.named_sequence @le_1(%arg0: !transform.any_op {transform.readonly}) -> !transform.any_op { transform.match.operation_name %arg0 ["func.func"] : !transform.any_op %0 = transform.test_produce_param_with_number_of_test_ops %arg0 : !transform.any_op @@ -1634,8 +1819,7 @@ module attributes { transform.with_named_sequence } { transform.yield } - transform.sequence failures(propagate) { - ^bb(%arg0: !transform.any_op): + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { %0 = transform.foreach_match in %arg0 @eq_1 -> @do_nothing : (!transform.any_op) -> !transform.any_op %1 = transform.foreach_match in %0 @ne_0 -> @do_nothing : (!transform.any_op) -> !transform.any_op %2 = transform.foreach_match in %1 @gt_m1 -> @do_nothing : (!transform.any_op) -> !transform.any_op @@ -1675,47 +1859,49 @@ func.func @test_tracked_rewrite() { func.return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %0 = transform.structured.match ops{["transform.test_dummy_payload_op"]} in %arg1 : (!transform.any_op) -> !transform.any_op - // expected-remark @below {{2 iterations}} - transform.test_tracked_rewrite %0 : (!transform.any_op) -> () - // One replacement op (test.drop_mapping) is dropped from the mapping. - %p = num_associations %0 : (!transform.any_op) -> !transform.param - // expected-remark @below {{2}} - test_print_param %p : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %0 = transform.structured.match ops{["transform.test_dummy_payload_op"]} in %arg1 : (!transform.any_op) -> !transform.any_op + // expected-remark @below {{2 iterations}} + transform.test_tracked_rewrite %0 : (!transform.any_op) -> () + // One replacement op (test.drop_mapping) is dropped from the mapping. + %p = transform.num_associations %0 : (!transform.any_op) -> !transform.param + // expected-remark @below {{2}} + transform.test_print_param %p : !transform.param + transform.yield + } } // ----- // Parameter deduplication happens by value -module { +module attributes {transform.with_named_sequence} { - transform.sequence failures(propagate) { - ^bb0(%0: !transform.any_op): + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { %1 = transform.param.constant 1 -> !transform.param %2 = transform.param.constant 1 -> !transform.param %3 = transform.param.constant 2 -> !transform.param %4 = transform.merge_handles %1, %2 { deduplicate } : !transform.param - %p = num_associations %4 : (!transform.param) -> !transform.param + %p = transform.num_associations %4 : (!transform.param) -> !transform.param // expected-remark @below {{1}} - test_print_param %p : !transform.param + transform.test_print_param %p : !transform.param %5 = transform.merge_handles %1, %1 { deduplicate } : !transform.param - %p2 = num_associations %5 : (!transform.param) -> !transform.param + %p2 = transform.num_associations %5 : (!transform.param) -> !transform.param // expected-remark @below {{1}} - test_print_param %p2 : !transform.param + transform.test_print_param %p2 : !transform.param %6 = transform.merge_handles %1, %3 { deduplicate } : !transform.param - %p3 = num_associations %6 : (!transform.param) -> !transform.param + %p3 = transform.num_associations %6 : (!transform.param) -> !transform.param // expected-remark @below {{2}} - test_print_param %p3 : !transform.param + transform.test_print_param %p3 : !transform.param %7 = transform.merge_handles %1, %1, %2, %3 : !transform.param - %p4 = num_associations %7 : (!transform.param) -> !transform.param + %p4 = transform.num_associations %7 : (!transform.param) -> !transform.param // expected-remark @below {{4}} - test_print_param %p4 : !transform.param + transform.test_print_param %p4 : !transform.param + transform.yield } } @@ -1723,32 +1909,34 @@ module { %0:3 = "test.get_two_results"() : () -> (i32, i32, f32) -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %1 = transform.structured.match ops{["test.get_two_results"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %2 = test_produce_value_handle_to_result %1, 0 : (!transform.any_op) -> !transform.any_value - %3 = test_produce_value_handle_to_result %1, 1 : (!transform.any_op) -> !transform.any_value +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %1 = transform.structured.match ops{["test.get_two_results"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %2 = transform.test_produce_value_handle_to_result %1, 0 : (!transform.any_op) -> !transform.any_value + %3 = transform.test_produce_value_handle_to_result %1, 1 : (!transform.any_op) -> !transform.any_value - %4 = transform.merge_handles %2, %2 { deduplicate } : !transform.any_value - %p = num_associations %4 : (!transform.any_value) -> !transform.param - // expected-remark @below {{1}} - test_print_param %p : !transform.param + %4 = transform.merge_handles %2, %2 { deduplicate } : !transform.any_value + %p = transform.num_associations %4 : (!transform.any_value) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p : !transform.param - %5 = transform.merge_handles %2, %3 { deduplicate } : !transform.any_value - %p2 = num_associations %5 : (!transform.any_value) -> !transform.param - // expected-remark @below {{2}} - test_print_param %p2 : !transform.param + %5 = transform.merge_handles %2, %3 { deduplicate } : !transform.any_value + %p2 = transform.num_associations %5 : (!transform.any_value) -> !transform.param + // expected-remark @below {{2}} + transform.test_print_param %p2 : !transform.param - %6 = test_produce_value_handle_to_result %1, 0 : (!transform.any_op) -> !transform.any_value - %7 = transform.merge_handles %2, %6 { deduplicate } : !transform.any_value - %p3 = num_associations %6 : (!transform.any_value) -> !transform.param - // expected-remark @below {{1}} - test_print_param %p3 : !transform.param + %6 = transform.test_produce_value_handle_to_result %1, 0 : (!transform.any_op) -> !transform.any_value + %7 = transform.merge_handles %2, %6 { deduplicate } : !transform.any_value + %p3 = transform.num_associations %6 : (!transform.any_value) -> !transform.param + // expected-remark @below {{1}} + transform.test_print_param %p3 : !transform.param - %8 = transform.merge_handles %2, %2, %3, %4 : !transform.any_value - %p4 = num_associations %8 : (!transform.any_value) -> !transform.param - // expected-remark @below {{4}} - test_print_param %p4 : !transform.param + %8 = transform.merge_handles %2, %2, %3, %4 : !transform.any_value + %p4 = transform.num_associations %8 : (!transform.any_value) -> !transform.param + // expected-remark @below {{4}} + transform.test_print_param %p4 : !transform.param + transform.yield + } } // ----- @@ -1775,18 +1963,20 @@ func.func @test_annotation() { %2 = "test.annotate_me"() {new_attr = 0} : () -> (i1) } -transform.sequence failures(propagate) { -^bb1(%arg0: !transform.any_op): - %0 = transform.structured.match ops{["test.annotate_me"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %1 = transform.test_produce_param_with_number_of_test_ops %0 : !transform.any_op - transform.annotate %0 "new_attr" = %1 : !transform.any_op, !transform.test_dialect_param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.structured.match ops{["test.annotate_me"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %1 = transform.test_produce_param_with_number_of_test_ops %0 : !transform.any_op + transform.annotate %0 "new_attr" = %1 : !transform.any_op, !transform.test_dialect_param - %2 = transform.param.constant 2 -> !transform.param - transform.annotate %0 "broadcast_attr" = %2 : !transform.any_op, !transform.param - transform.annotate %0 "unit_attr" : !transform.any_op + %2 = transform.param.constant 2 -> !transform.param + transform.annotate %0 "broadcast_attr" = %2 : !transform.any_op, !transform.param + transform.annotate %0 "unit_attr" : !transform.any_op - %3 = transform.param.constant "example" -> !transform.any_param - transform.annotate %0 "any_attr" = %3 : !transform.any_op, !transform.any_param + %3 = transform.param.constant "example" -> !transform.any_param + transform.annotate %0 "any_attr" = %3 : !transform.any_op, !transform.any_param + transform.yield + } } // ----- @@ -1798,12 +1988,14 @@ func.func @notify_payload_op_replaced(%arg0: index, %arg1: index) { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %0 = transform.structured.match attributes{original} in %arg1 : (!transform.any_op) -> !transform.any_op - %1 = transform.structured.match attributes{replacement} in %arg1 : (!transform.any_op) -> !transform.any_op - test_notify_payload_op_replaced %0, %1 : (!transform.any_op, !transform.any_op) -> () - test_print_remark_at_operand %0, "updated handle" : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %0 = transform.structured.match attributes{original} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.structured.match attributes{replacement} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_notify_payload_op_replaced %0, %1 : (!transform.any_op, !transform.any_op) -> () + transform.test_print_remark_at_operand %0, "updated handle" : !transform.any_op + transform.yield + } } // ----- @@ -1832,47 +2024,49 @@ func.func @test_apply_cse() -> (index, index, index) { return %0, %1, %3 : index, index, index } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %first = transform.structured.match attributes{first} in %0 : (!transform.any_op) -> !transform.any_op - %elim_first = transform.structured.match ops{["arith.constant"]} in %first : (!transform.any_op) -> !transform.any_op - %second = transform.structured.match attributes{first} in %0 : (!transform.any_op) -> !transform.any_op - %elim_second = transform.structured.match ops{["arith.constant"]} in %first : (!transform.any_op) -> !transform.any_op - - // There are 3 arith.constant ops. - %all = transform.structured.match ops{["arith.constant"]} in %0 : (!transform.any_op) -> !transform.any_op - %p = num_associations %all : (!transform.any_op) -> !transform.param - // expected-remark @below{{3}} - test_print_param %p : !transform.param - // "deduplicate" has no effect because these are 3 different ops. - %merged_before = transform.merge_handles deduplicate %all : !transform.any_op - %p2 = num_associations %merged_before : (!transform.any_op) -> !transform.param - // expected-remark @below{{3}} - test_print_param %p2 : !transform.param - - // Apply CSE. - transform.apply_cse to %0 : !transform.any_op - - // The handle is still mapped to 3 arith.constant ops. - %p3 = num_associations %all : (!transform.any_op) -> !transform.param - // expected-remark @below{{3}} - test_print_param %p3 : !transform.param - // But they are all the same op. - %merged_after = transform.merge_handles deduplicate %all : !transform.any_op - %p4 = num_associations %merged_after : (!transform.any_op) -> !transform.param - // expected-remark @below{{1}} - test_print_param %p4 : !transform.param - - // The other handles were also updated. - test_print_remark_at_operand %elim_first, "eliminated 1" : !transform.any_op - %p5 = num_associations %elim_first : (!transform.any_op) -> !transform.param - // expected-remark @below{{1}} - test_print_param %p5 : !transform.param - test_print_remark_at_operand %elim_second, "eliminated 2" : !transform.any_op - %p6 = num_associations %elim_second : (!transform.any_op) -> !transform.param - // expected-remark @below{{1}} - test_print_param %p6 : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %0 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %first = transform.structured.match attributes{first} in %0 : (!transform.any_op) -> !transform.any_op + %elim_first = transform.structured.match ops{["arith.constant"]} in %first : (!transform.any_op) -> !transform.any_op + %second = transform.structured.match attributes{first} in %0 : (!transform.any_op) -> !transform.any_op + %elim_second = transform.structured.match ops{["arith.constant"]} in %first : (!transform.any_op) -> !transform.any_op + + // There are 3 arith.constant ops. + %all = transform.structured.match ops{["arith.constant"]} in %0 : (!transform.any_op) -> !transform.any_op + %p = transform.num_associations %all : (!transform.any_op) -> !transform.param + // expected-remark @below{{3}} + transform.test_print_param %p : !transform.param + // "deduplicate" has no effect because these are 3 different ops. + %merged_before = transform.merge_handles deduplicate %all : !transform.any_op + %p2 = transform.num_associations %merged_before : (!transform.any_op) -> !transform.param + // expected-remark @below{{3}} + transform.test_print_param %p2 : !transform.param + + // Apply CSE. + transform.apply_cse to %0 : !transform.any_op + + // The handle is still mapped to 3 arith.constant ops. + %p3 = transform.num_associations %all : (!transform.any_op) -> !transform.param + // expected-remark @below{{3}} + transform.test_print_param %p3 : !transform.param + // But they are all the same op. + %merged_after = transform.merge_handles deduplicate %all : !transform.any_op + %p4 = transform.num_associations %merged_after : (!transform.any_op) -> !transform.param + // expected-remark @below{{1}} + transform.test_print_param %p4 : !transform.param + + // The other handles were also updated. + transform.test_print_remark_at_operand %elim_first, "eliminated 1" : !transform.any_op + %p5 = transform.num_associations %elim_first : (!transform.any_op) -> !transform.param + // expected-remark @below{{1}} + transform.test_print_param %p5 : !transform.param + transform.test_print_remark_at_operand %elim_second, "eliminated 2" : !transform.any_op + %p6 = transform.num_associations %elim_second : (!transform.any_op) -> !transform.param + // expected-remark @below{{1}} + transform.test_print_param %p6 : !transform.param + transform.yield + } } // ----- @@ -1890,24 +2084,26 @@ func.func @test_licm(%arg0: index, %arg1: index, %arg2: index) { return } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %0 = transform.structured.match ops{["scf.for"]} in %arg1 : (!transform.any_op) -> !transform.any_op - transform.apply_licm to %0 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %0 = transform.structured.match ops{["scf.for"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.apply_licm to %0 : !transform.any_op + transform.yield + } } // ----- // expected-note @below{{when applied to this op}} -module { +module attributes {transform.with_named_sequence} { func.func @test_licm_invalid() { return } - transform.sequence failures(propagate) { - ^bb1(%arg1: !transform.any_op): + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { // expected-error @below{{transform applied to the wrong op kind}} transform.apply_licm to %arg1 : !transform.any_op + transform.yield } } @@ -1924,38 +2120,40 @@ func.func @get_parent_op() { }) : () -> () } -transform.sequence failures(propagate) { -^bb1(%arg1: !transform.any_op): - %0 = transform.structured.match ops{["test.qux"]} in %arg1 : (!transform.any_op) -> !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op) { + %0 = transform.structured.match ops{["test.qux"]} in %arg1 : (!transform.any_op) -> !transform.any_op - // Get parent by name. - %1 = transform.get_parent_op %0 {op_name = "test.foo"} : (!transform.any_op) -> !transform.any_op - test_print_remark_at_operand %1, "found test.foo parent" : !transform.any_op + // Get parent by name. + %1 = transform.get_parent_op %0 {op_name = "test.foo"} : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %1, "found test.foo parent" : !transform.any_op - // Get immediate parent. - %2 = transform.get_parent_op %0 : (!transform.any_op) -> !transform.any_op - test_print_remark_at_operand %2, "direct parent" : !transform.any_op - %p = num_associations %2 : (!transform.any_op) -> !transform.param - // expected-remark @below{{2}} - test_print_param %p : !transform.param + // Get immediate parent. + %2 = transform.get_parent_op %0 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %2, "direct parent" : !transform.any_op + %p = transform.num_associations %2 : (!transform.any_op) -> !transform.param + // expected-remark @below{{2}} + transform.test_print_param %p : !transform.param - // Deduplicate results. - %3 = transform.structured.match ops{["test.qux"]} in %arg1 : (!transform.any_op) -> !transform.any_op - %4 = transform.get_parent_op %3 {deduplicate} : (!transform.any_op) -> !transform.any_op - %p2 = num_associations %4 : (!transform.any_op) -> !transform.param - // expected-remark @below{{1}} - test_print_param %p2 : !transform.param + // Deduplicate results. + %3 = transform.structured.match ops{["test.qux"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %4 = transform.get_parent_op %3 {deduplicate} : (!transform.any_op) -> !transform.any_op + %p2 = transform.num_associations %4 : (!transform.any_op) -> !transform.param + // expected-remark @below{{1}} + transform.test_print_param %p2 : !transform.param + transform.yield + } } // ----- // expected-note @below {{target op}} -module { - transform.sequence failures(propagate) { - ^bb0(%arg0: !transform.any_op): +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { // expected-error @below{{could not find a parent op that matches all requirements}} - %3 = get_parent_op %arg0 {op_name = "builtin.module"} : (!transform.any_op) -> !transform.any_op + %3 = transform.get_parent_op %arg0 {op_name = "builtin.module"} : (!transform.any_op) -> !transform.any_op + transform.yield } } @@ -1967,29 +2165,34 @@ func.func @cast(%arg0: f32) -> f64 { return %0 : f64 } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.structured.match ops{["arith.extf"]} in %arg0 : (!transform.any_op) -> !transform.op<"arith.extf"> - %1 = transform.get_result %0[0] : (!transform.op<"arith.extf">) -> !transform.any_value - %2 = transform.get_type %1 : (!transform.any_value) -> !transform.type - transform.test_print_param %2 at %0 : !transform.type, !transform.op<"arith.extf"> - transform.yield +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.structured.match ops{["arith.extf"]} in %arg0 : (!transform.any_op) -> !transform.op<"arith.extf"> + %1 = transform.get_result %0[0] : (!transform.op<"arith.extf">) -> !transform.any_value + %2 = transform.get_type %1 : (!transform.any_value) -> !transform.type + transform.test_print_param %2 at %0 : !transform.type, !transform.op<"arith.extf"> + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-error @below {{expected type attribute, got 0 : i32}} - transform.test_produce_param (0 : i32) : !transform.type +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-error @below {{expected type attribute, got 0 : i32}} + transform.test_produce_param (0 : i32) : !transform.type + transform.yield + } } // ----- -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // expected-error @below {{expected affine map attribute, got 0 : i32}} - transform.test_produce_param (0 : i32) : !transform.affine_map +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // expected-error @below {{expected affine map attribute, got 0 : i32}} + transform.test_produce_param (0 : i32) : !transform.affine_map + transform.yield + } } // ----- @@ -1997,10 +2200,12 @@ transform.sequence failures(propagate) { // CHECK-LABEL: @type_param_anchor func.func private @type_param_anchor() -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // CHECK: test_produce_param(f32) : !transform.type - transform.test_produce_param(f32) : !transform.type +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // CHECK: test_produce_param(f32) : !transform.type + transform.test_produce_param(f32) : !transform.type + transform.yield + } } // ----- @@ -2008,10 +2213,12 @@ transform.sequence failures(propagate) { // CHECK-LABEL: @affine_map_param_anchor func.func private @affine_map_param_anchor() -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // CHECK: test_produce_param(#{{.*}}) : !transform.affine_map - transform.test_produce_param(affine_map<(d0) -> ()>) : !transform.affine_map +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // CHECK: test_produce_param(#{{.*}}) : !transform.affine_map + transform.test_produce_param(affine_map<(d0) -> ()>) : !transform.affine_map + transform.yield + } } // ----- @@ -2020,10 +2227,12 @@ func.func @verify_success(%arg0: f64) -> f64 { return %arg0 : f64 } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op - transform.verify %0 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op + transform.verify %0 : !transform.any_op + transform.yield + } } // ----- @@ -2034,12 +2243,14 @@ func.func @verify_failure(%arg0: f64) -> f64 { return %arg0 : f64 } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %0 = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op - transform.test_produce_invalid_ir %0 : !transform.any_op - // expected-error @below{{failed to verify payload op}} - transform.verify %0 : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %0 = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op + transform.test_produce_invalid_ir %0 : !transform.any_op + // expected-error @below{{failed to verify payload op}} + transform.verify %0 : !transform.any_op + transform.yield + } } // ----- @@ -2054,22 +2265,24 @@ func.func @select() { func.return } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - // Match all ops inside the function (including the function itself). - %func_op = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %0 = transform.structured.match in %func_op : (!transform.any_op) -> !transform.any_op - %p = num_associations %0 : (!transform.any_op) -> !transform.param - // expected-remark @below{{5}} - test_print_param %p : !transform.param +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // Match all ops inside the function (including the function itself). + %func_op = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %0 = transform.structured.match in %func_op : (!transform.any_op) -> !transform.any_op + %p = transform.num_associations %0 : (!transform.any_op) -> !transform.param + // expected-remark @below{{5}} + transform.test_print_param %p : !transform.param - // Select "test.foo". - %foo = transform.select "test.foo" in %0 : (!transform.any_op) -> !transform.any_op - test_print_remark_at_operand %foo, "found foo" : !transform.any_op + // Select "test.foo". + %foo = transform.select "test.foo" in %0 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %foo, "found foo" : !transform.any_op - // Select "test.bar". - %bar = transform.select "test.bar" in %0 : (!transform.any_op) -> !transform.any_op - test_print_remark_at_operand %bar, "found bar" : !transform.any_op + // Select "test.bar". + %bar = transform.select "test.bar" in %0 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %bar, "found bar" : !transform.any_op + transform.yield + } } // ----- @@ -2085,15 +2298,17 @@ func.func @apply_dce(%f: f32, %m: memref<5xf32>, %idx: index) { return } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - %func_op = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op - %empty_op = transform.structured.match ops{["tensor.empty"]} in %func_op : (!transform.any_op) -> !transform.any_op - transform.apply_dce to %func_op : !transform.any_op +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + %func_op = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %empty_op = transform.structured.match ops{["tensor.empty"]} in %func_op : (!transform.any_op) -> !transform.any_op + transform.apply_dce to %func_op : !transform.any_op - %p = num_associations %empty_op : (!transform.any_op) -> !transform.param - // expected-remark @below{{0}} - test_print_param %p : !transform.param + %p = transform.num_associations %empty_op : (!transform.any_op) -> !transform.param + // expected-remark @below{{0}} + transform.test_print_param %p : !transform.param + transform.yield + } } @@ -2110,27 +2325,26 @@ module @named_inclusion attributes { transform.with_named_sequence } { // Match `arith.constant`s that are not nested under a `scf.for` and ensure // there are none in the program -transform.named_sequence @print(%root: !transform.any_op {transform.readonly}) { - transform.test_print_remark_at_operand %root, "matched func" : !transform.any_op - transform.yield -} + transform.named_sequence @print(%root: !transform.any_op {transform.readonly}) { + transform.test_print_remark_at_operand %root, "matched func" : !transform.any_op + transform.yield + } -transform.named_sequence @match_constant_not_under_scf_for(%root: !transform.any_op {transform.readonly}) - -> !transform.any_op { - transform.match.operation_name %root ["arith.constant"] : !transform.any_op - %for = transform.get_parent_op %root { op_name = "scf.for", allow_empty_results } - : (!transform.any_op) -> (!transform.any_op) - transform.match.operation_empty %for : !transform.any_op - transform.yield %root : !transform.any_op -} + transform.named_sequence @match_constant_not_under_scf_for(%root: !transform.any_op {transform.readonly}) + -> !transform.any_op { + transform.match.operation_name %root ["arith.constant"] : !transform.any_op + %for = transform.get_parent_op %root { op_name = "scf.for", allow_empty_results } + : (!transform.any_op) -> (!transform.any_op) + transform.match.operation_empty %for : !transform.any_op + transform.yield %root : !transform.any_op + } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - transform.foreach_match in %arg0 - @match_constant_not_under_scf_for -> @print - : (!transform.any_op) -> (!transform.any_op) - transform.yield -} + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + transform.foreach_match in %arg0 + @match_constant_not_under_scf_for -> @print + : (!transform.any_op) -> (!transform.any_op) + transform.yield + } } // ----- @@ -2142,28 +2356,27 @@ func.func @no_constant_under_loop(%lb: index, %ub: index, %step: index) { } module @named_inclusion attributes { transform.with_named_sequence } { -// Match `arith.constant`s that are not nested under a `scf.for` and ensure -// there are none in the program + // Match `arith.constant`s that are not nested under a `scf.for` and ensure + // there are none in the program -transform.named_sequence @print(%root: !transform.any_op {transform.readonly}) { - transform.test_print_remark_at_operand %root, "no parent scf.for" : !transform.any_op - transform.yield -} + transform.named_sequence @print(%root: !transform.any_op {transform.readonly}) { + transform.test_print_remark_at_operand %root, "no parent scf.for" : !transform.any_op + transform.yield + } -transform.named_sequence @match_constant_not_under_scf_for(%root: !transform.any_op {transform.readonly}) - -> !transform.any_op { - transform.match.operation_name %root ["arith.constant"] : !transform.any_op - %for = transform.get_parent_op %root { op_name = "scf.for", allow_empty_results } - : (!transform.any_op) -> (!transform.any_op) - transform.match.operation_empty %for : !transform.any_op - transform.yield %root : !transform.any_op -} + transform.named_sequence @match_constant_not_under_scf_for(%root: !transform.any_op {transform.readonly}) + -> !transform.any_op { + transform.match.operation_name %root ["arith.constant"] : !transform.any_op + %for = transform.get_parent_op %root { op_name = "scf.for", allow_empty_results } + : (!transform.any_op) -> (!transform.any_op) + transform.match.operation_empty %for : !transform.any_op + transform.yield %root : !transform.any_op + } -transform.sequence failures(propagate) { -^bb0(%arg0: !transform.any_op): - transform.foreach_match in %arg0 - @match_constant_not_under_scf_for -> @print - : (!transform.any_op) -> (!transform.any_op) - transform.yield -} + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + transform.foreach_match in %arg0 + @match_constant_not_under_scf_for -> @print + : (!transform.any_op) -> (!transform.any_op) + transform.yield + } } -- GitLab From 2ab5c47c8752b444885d6bfaf6f570a482fb4cdf Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 4 Jan 2024 20:39:44 +0000 Subject: [PATCH 036/728] [VPlan] Don't replace scalarizing recipe with VPWidenCastRecipe. Don't replace a scalarizing recipe with a VPWidenCastRecipe. This would introduce wide (vectorizing) recipes when interleaving only. Fixes https://github.com/llvm/llvm-project/issues/76986 --- .../Transforms/Vectorize/VPlanTransforms.cpp | 23 ++++++---- .../interleave-and-scalarize-only.ll | 45 +++++++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 33132880d5a4..5c430620a2dc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -829,15 +829,20 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { Type *ATy = TypeInfo.inferScalarType(A); if (TruncTy == ATy) { Trunc->replaceAllUsesWith(A); - } else if (ATy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) { - auto *VPC = - new VPWidenCastRecipe(Instruction::CastOps(ExtOpcode), A, TruncTy); - VPC->insertBefore(&R); - Trunc->replaceAllUsesWith(VPC); - } else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) { - auto *VPC = new VPWidenCastRecipe(Instruction::Trunc, A, TruncTy); - VPC->insertBefore(&R); - Trunc->replaceAllUsesWith(VPC); + } else { + // Don't replace a scalarizing recipe with a widened cast. + if (isa(&R)) + break; + if (ATy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) { + auto *VPC = + new VPWidenCastRecipe(Instruction::CastOps(ExtOpcode), A, TruncTy); + VPC->insertBefore(&R); + Trunc->replaceAllUsesWith(VPC); + } else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) { + auto *VPC = new VPWidenCastRecipe(Instruction::Trunc, A, TruncTy); + VPC->insertBefore(&R); + Trunc->replaceAllUsesWith(VPC); + } } #ifndef NDEBUG // Verify that the cached type info is for both A and its users is still diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index 7b9d7f798639..297cd2a7c12f 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -310,3 +310,48 @@ loop: exit: ret void } + +define void @pr76986_trunc_sext_interleaving_only(i16 %arg, ptr noalias %src, ptr noalias %dst) { +; CHECK-LABEL: define void @pr76986_trunc_sext_interleaving_only( +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr %src, i64 [[TMP0]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr %src, i64 [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP2]], align 1 +; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1 +; CHECK-NEXT: [[TMP6:%.*]] = sext i8 [[TMP4]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = sext i8 [[TMP5]] to i32 +; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP6]] to i16 +; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP7]] to i16 +; CHECK-NEXT: [[TMP10:%.*]] = sdiv i16 [[TMP8]], %arg +; CHECK-NEXT: [[TMP11:%.*]] = sdiv i16 [[TMP9]], %arg +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr %dst, i64 [[TMP0]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i16, ptr %dst, i64 [[TMP1]] +; CHECK-NEXT: store i16 [[TMP10]], ptr [[TMP12]], align 2 +; CHECK-NEXT: store i16 [[TMP11]], ptr [[TMP13]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 14934 +; CHECK-NEXT: br i1 [[TMP14]], label %middle.block, label %vector.body +; +bb: + br label %loop + +loop: + %iv = phi i64 [ 0, %bb ], [ %iv.next, %loop ] + %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv + %l = load i8, ptr %gep.src + %sext = sext i8 %l to i32 + %trunc = trunc i32 %sext to i16 + %sdiv = sdiv i16 %trunc, %arg + %gep.dst = getelementptr inbounds i16, ptr %dst, i64 %iv + store i16 %sdiv, ptr %gep.dst + %iv.next = add i64 %iv, 1 + %icmp = icmp ult i64 %iv, 14933 + br i1 %icmp, label %loop, label %exit + +exit: + ret void +} + -- GitLab From 58f1640635feff282935153d295dfb4ea1818401 Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Thu, 4 Jan 2024 15:40:30 -0500 Subject: [PATCH 037/728] [RISCV][llvm-mca] Use correct LMUL and SEW for strided loads and stores (#76869) The pseudos for strided loads and stores use the SEW coming from the name. For example, vlse8 has SEW=8 and vlse16 has SEW=16. When llvm-mca tries to lookup (VLSE8_V, SEW=S, LMUL=L) in the inverse pseudo table, a result will only be found when S=8, where S was set from the previous vsetvli instruction. Instead, for a match to be found, we must lookup (VLSE8_V, SEW=8, LMUL=L') where L' is the EMUL which was calculated by scaling the LMUL and SEW from the previous vsetvli and the SEW=8. --- .../Target/RISCV/MCA/RISCVCustomBehaviour.cpp | 33 +- .../RISCV/SiFive7/strided-load-store.s | 361 ++++++++++++++++++ 2 files changed, 385 insertions(+), 9 deletions(-) create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-store.s diff --git a/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp b/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp index aba2511959af..8d97c5ffd20a 100644 --- a/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp +++ b/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp @@ -186,30 +186,37 @@ RISCVInstrumentManager::createInstruments(const MCInst &Inst) { } static std::pair -getEEWAndEMULForUnitStrideLoadStore(unsigned Opcode, RISCVII::VLMUL LMUL, - uint8_t SEW) { +getEEWAndEMUL(unsigned Opcode, RISCVII::VLMUL LMUL, uint8_t SEW) { uint8_t EEW; switch (Opcode) { case RISCV::VLM_V: case RISCV::VSM_V: case RISCV::VLE8_V: case RISCV::VSE8_V: + case RISCV::VLSE8_V: + case RISCV::VSSE8_V: EEW = 8; break; case RISCV::VLE16_V: case RISCV::VSE16_V: + case RISCV::VLSE16_V: + case RISCV::VSSE16_V: EEW = 16; break; case RISCV::VLE32_V: case RISCV::VSE32_V: + case RISCV::VLSE32_V: + case RISCV::VSSE32_V: EEW = 32; break; case RISCV::VLE64_V: case RISCV::VSE64_V: + case RISCV::VLSE64_V: + case RISCV::VSSE64_V: EEW = 64; break; default: - llvm_unreachable("Opcode is not a vector unit stride load nor store"); + llvm_unreachable("Could not determine EEW from Opcode"); } auto EMUL = RISCVVType::getSameRatioLMUL(SEW, LMUL, EEW); @@ -218,6 +225,18 @@ getEEWAndEMULForUnitStrideLoadStore(unsigned Opcode, RISCVII::VLMUL LMUL, return std::make_pair(EEW, *EMUL); } +bool opcodeHasEEWAndEMULInfo(unsigned short Opcode) { + return Opcode == RISCV::VLM_V || Opcode == RISCV::VSM_V || + Opcode == RISCV::VLE8_V || Opcode == RISCV::VSE8_V || + Opcode == RISCV::VLE16_V || Opcode == RISCV::VSE16_V || + Opcode == RISCV::VLE32_V || Opcode == RISCV::VSE32_V || + Opcode == RISCV::VLE64_V || Opcode == RISCV::VSE64_V || + Opcode == RISCV::VLSE8_V || Opcode == RISCV::VSSE8_V || + Opcode == RISCV::VLSE16_V || Opcode == RISCV::VSSE16_V || + Opcode == RISCV::VLSE32_V || Opcode == RISCV::VSSE32_V || + Opcode == RISCV::VLSE64_V || Opcode == RISCV::VSSE64_V; +} + unsigned RISCVInstrumentManager::getSchedClassID( const MCInstrInfo &MCII, const MCInst &MCI, const llvm::SmallVector &IVec) const { @@ -249,13 +268,9 @@ unsigned RISCVInstrumentManager::getSchedClassID( uint8_t SEW = SI ? SI->getSEW() : 0; const RISCVVInversePseudosTable::PseudoInfo *RVV = nullptr; - if (Opcode == RISCV::VLM_V || Opcode == RISCV::VSM_V || - Opcode == RISCV::VLE8_V || Opcode == RISCV::VSE8_V || - Opcode == RISCV::VLE16_V || Opcode == RISCV::VSE16_V || - Opcode == RISCV::VLE32_V || Opcode == RISCV::VSE32_V || - Opcode == RISCV::VLE64_V || Opcode == RISCV::VSE64_V) { + if (opcodeHasEEWAndEMULInfo(Opcode)) { RISCVII::VLMUL VLMUL = static_cast(LMUL); - auto [EEW, EMUL] = getEEWAndEMULForUnitStrideLoadStore(Opcode, VLMUL, SEW); + auto [EEW, EMUL] = getEEWAndEMUL(Opcode, VLMUL, SEW); RVV = RISCVVInversePseudosTable::getBaseInfo(Opcode, EMUL, EEW); } else { // Check if it depends on LMUL and SEW diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-store.s b/llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-store.s new file mode 100644 index 000000000000..d60922b06462 --- /dev/null +++ b/llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-store.s @@ -0,0 +1,361 @@ +# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py +# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-x280 -iterations=1 < %s | FileCheck %s + +vsetvli zero, zero, e8, mf8, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e8, mf4, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e8, mf2, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e8, m1, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e8, m2, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 + +vsetvli zero, zero, e8, m4, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 + +vsetvli zero, zero, e8, m8, tu, mu +vlse8.v v1, (a1), a2 + +vsetvli zero, zero, e16, mf4, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e16, mf2, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e16, m1, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e16, m2, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e16, m4, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 + +vsetvli zero, zero, e16, m8, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 + +vsetvli zero, zero, e32, mf2, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e32, m1, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e32, m2, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e32, m4, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e32, m8, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 + +vsetvli zero, zero, e64, m1, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e64, m2, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e64, m4, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +vsetvli zero, zero, e64, m8, tu, mu +vlse8.v v1, (a1), a2 +vlse16.v v1, (a1), a2 +vlse32.v v1, (a1), a2 +vlse64.v v1, (a1), a2 + +# CHECK: Iterations: 1 +# CHECK-NEXT: Instructions: 100 +# CHECK-NEXT: Total Cycles: 4734 +# CHECK-NEXT: Total uOps: 100 + +# CHECK: Dispatch Width: 2 +# CHECK-NEXT: uOps Per Cycle: 0.02 +# CHECK-NEXT: IPC: 0.02 +# CHECK-NEXT: Block RThroughput: 4686.0 + +# CHECK: Instruction Info: +# CHECK-NEXT: [1]: #uOps +# CHECK-NEXT: [2]: Latency +# CHECK-NEXT: [3]: RThroughput +# CHECK-NEXT: [4]: MayLoad +# CHECK-NEXT: [5]: MayStore +# CHECK-NEXT: [6]: HasSideEffects (U) + +# CHECK: [1] [2] [3] [4] [5] [6] Instructions: +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf8, tu, mu +# CHECK-NEXT: 1 11 9.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf4, tu, mu +# CHECK-NEXT: 1 19 17.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, mf2, tu, mu +# CHECK-NEXT: 1 35 33.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, m1, tu, mu +# CHECK-NEXT: 1 67 65.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, m2, tu, mu +# CHECK-NEXT: 1 131 129.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 131 129.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 131 129.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, m4, tu, mu +# CHECK-NEXT: 1 259 257.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 259 257.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e8, m8, tu, mu +# CHECK-NEXT: 1 515 513.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, mf4, tu, mu +# CHECK-NEXT: 1 11 9.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, mf2, tu, mu +# CHECK-NEXT: 1 19 17.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, m1, tu, mu +# CHECK-NEXT: 1 35 33.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, m2, tu, mu +# CHECK-NEXT: 1 67 65.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, m4, tu, mu +# CHECK-NEXT: 1 131 129.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 131 129.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 131 129.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e16, m8, tu, mu +# CHECK-NEXT: 1 259 257.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 259 257.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e32, mf2, tu, mu +# CHECK-NEXT: 1 11 9.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e32, m1, tu, mu +# CHECK-NEXT: 1 19 17.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e32, m2, tu, mu +# CHECK-NEXT: 1 35 33.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e32, m4, tu, mu +# CHECK-NEXT: 1 67 65.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e32, m8, tu, mu +# CHECK-NEXT: 1 131 129.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 131 129.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 131 129.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e64, m1, tu, mu +# CHECK-NEXT: 1 11 9.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e64, m2, tu, mu +# CHECK-NEXT: 1 19 17.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e64, m4, tu, mu +# CHECK-NEXT: 1 35 33.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 35 33.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e64, m8, tu, mu +# CHECK-NEXT: 1 67 65.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 67 65.00 * vlse64.v v1, (a1), a2 + +# CHECK: Resources: +# CHECK-NEXT: [0] - SiFive7FDiv +# CHECK-NEXT: [1] - SiFive7IDiv +# CHECK-NEXT: [2] - SiFive7PipeA +# CHECK-NEXT: [3] - SiFive7PipeB +# CHECK-NEXT: [4] - SiFive7VA +# CHECK-NEXT: [5] - SiFive7VCQ +# CHECK-NEXT: [6] - SiFive7VL +# CHECK-NEXT: [7] - SiFive7VS + +# CHECK: Resource pressure per iteration: +# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] +# CHECK-NEXT: - - 22.00 - - 78.00 4686.00 - + +# CHECK: Resource pressure by instruction: +# CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] Instructions: +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf8, tu, mu +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf4, tu, mu +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, mf2, tu, mu +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, m1, tu, mu +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, m2, tu, mu +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, m4, tu, mu +# CHECK-NEXT: - - - - - 1.00 257.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 257.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e8, m8, tu, mu +# CHECK-NEXT: - - - - - 1.00 513.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, mf4, tu, mu +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, mf2, tu, mu +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, m1, tu, mu +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, m2, tu, mu +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, m4, tu, mu +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e16, m8, tu, mu +# CHECK-NEXT: - - - - - 1.00 257.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 257.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e32, mf2, tu, mu +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e32, m1, tu, mu +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e32, m2, tu, mu +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e32, m4, tu, mu +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e32, m8, tu, mu +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 129.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e64, m1, tu, mu +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e64, m2, tu, mu +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e64, m4, tu, mu +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 33.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e64, m8, tu, mu +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 65.00 - vlse64.v v1, (a1), a2 -- GitLab From a0c19bd45599c044a662752c2b513cc514cb645a Mon Sep 17 00:00:00 2001 From: Maksim Levental Date: Thu, 4 Jan 2024 14:43:52 -0600 Subject: [PATCH 038/728] [mlir][RegionBranchOpInterface] explicitly check for existance of block terminator (#76831) --- mlir/lib/Interfaces/ControlFlowInterfaces.cpp | 7 +-- .../IR/test-region-branch-op-verifier.mlir | 15 +++++- mlir/test/lib/Dialect/Test/TestDialect.cpp | 50 +++++++++++++++++++ mlir/test/lib/Dialect/Test/TestOps.td | 12 +++++ 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp index 4ed024ddae24..a563ec5cb8db 100644 --- a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp +++ b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp @@ -177,9 +177,10 @@ LogicalResult detail::verifyTypesAlongControlFlowEdges(Operation *op) { SmallVector regionReturnOps; for (Block &block : region) - if (auto terminator = dyn_cast( - block.getTerminator())) - regionReturnOps.push_back(terminator); + if (!block.empty()) + if (auto terminator = + dyn_cast(block.back())) + regionReturnOps.push_back(terminator); // If there is no return-like terminator, the op itself should verify // type consistency. diff --git a/mlir/test/IR/test-region-branch-op-verifier.mlir b/mlir/test/IR/test-region-branch-op-verifier.mlir index f5fb7fc2b25c..b94f6beb9796 100644 --- a/mlir/test/IR/test-region-branch-op-verifier.mlir +++ b/mlir/test/IR/test-region-branch-op-verifier.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s +// RUN: mlir-opt %s -split-input-file func.func @test_ops_verify(%arg: i32) -> f32 { %0 = "test.constant"() { value = 5.3 : f32 } : () -> f32 @@ -8,3 +8,16 @@ func.func @test_ops_verify(%arg: i32) -> f32 { } return %1 : f32 } + +// ----- + +func.func @test_no_terminator(%arg: index) { + test.switch_with_no_break %arg + case 0 { + ^bb: + } + case 1 { + ^bb: + } + return +} diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp index a1b30705f16a..cb5ee6014b61 100644 --- a/mlir/test/lib/Dialect/Test/TestDialect.cpp +++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp @@ -53,6 +53,7 @@ using namespace test; Attribute MyPropStruct::asAttribute(MLIRContext *ctx) const { return StringAttr::get(ctx, content); } + LogicalResult MyPropStruct::setFromAttr(MyPropStruct &prop, Attribute attr, function_ref emitError) { @@ -64,6 +65,7 @@ MyPropStruct::setFromAttr(MyPropStruct &prop, Attribute attr, prop.content = strAttr.getValue(); return success(); } + llvm::hash_code MyPropStruct::hash() const { return hash_value(StringRef(content)); } @@ -127,6 +129,12 @@ static void customPrintProperties(OpAsmPrinter &p, const VersionedProperties &prop); static ParseResult customParseProperties(OpAsmParser &parser, VersionedProperties &prop); +static ParseResult +parseSwitchCases(OpAsmParser &p, DenseI64ArrayAttr &cases, + SmallVectorImpl> &caseRegions); + +static void printSwitchCases(OpAsmPrinter &p, Operation *op, + DenseI64ArrayAttr cases, RegionRange caseRegions); void test::registerTestDialect(DialectRegistry ®istry) { registry.insert(); @@ -230,6 +238,7 @@ void TestDialect::initialize() { // unregistered op. fallbackEffectOpInterfaces = new TestOpEffectInterfaceFallback; } + TestDialect::~TestDialect() { delete static_cast( fallbackEffectOpInterfaces); @@ -1013,6 +1022,13 @@ LoopBlockTerminatorOp::getMutableSuccessorOperands(RegionBranchPoint point) { return getNextIterArgMutable(); } +//===----------------------------------------------------------------------===// +// SwitchWithNoBreakOp +//===----------------------------------------------------------------------===// + +void TestNoTerminatorOp::getSuccessorRegions( + RegionBranchPoint point, SmallVectorImpl ®ions) {} + //===----------------------------------------------------------------------===// // SingleNoTerminatorCustomAsmOp //===----------------------------------------------------------------------===// @@ -1160,6 +1176,7 @@ setPropertiesFromAttribute(PropertiesWithCustomPrint &prop, Attribute attr, prop.value = valueAttr.getValue().getSExtValue(); return success(); } + static DictionaryAttr getPropertiesAsAttribute(MLIRContext *ctx, const PropertiesWithCustomPrint &prop) { @@ -1169,14 +1186,17 @@ getPropertiesAsAttribute(MLIRContext *ctx, attrs.push_back(b.getNamedAttr("value", b.getI32IntegerAttr(prop.value))); return b.getDictionaryAttr(attrs); } + static llvm::hash_code computeHash(const PropertiesWithCustomPrint &prop) { return llvm::hash_combine(prop.value, StringRef(*prop.label)); } + static void customPrintProperties(OpAsmPrinter &p, const PropertiesWithCustomPrint &prop) { p.printKeywordOrString(*prop.label); p << " is " << prop.value; } + static ParseResult customParseProperties(OpAsmParser &parser, PropertiesWithCustomPrint &prop) { std::string label; @@ -1186,6 +1206,31 @@ static ParseResult customParseProperties(OpAsmParser &parser, prop.label = std::make_shared(std::move(label)); return success(); } + +static ParseResult +parseSwitchCases(OpAsmParser &p, DenseI64ArrayAttr &cases, + SmallVectorImpl> &caseRegions) { + SmallVector caseValues; + while (succeeded(p.parseOptionalKeyword("case"))) { + int64_t value; + Region ®ion = *caseRegions.emplace_back(std::make_unique()); + if (p.parseInteger(value) || p.parseRegion(region, /*arguments=*/{})) + return failure(); + caseValues.push_back(value); + } + cases = p.getBuilder().getDenseI64ArrayAttr(caseValues); + return success(); +} + +static void printSwitchCases(OpAsmPrinter &p, Operation *op, + DenseI64ArrayAttr cases, RegionRange caseRegions) { + for (auto [value, region] : llvm::zip(cases.asArrayRef(), caseRegions)) { + p.printNewline(); + p << "case " << value << ' '; + p.printRegion(*region, /*printEntryBlockArgs=*/false); + } +} + static LogicalResult setPropertiesFromAttribute(VersionedProperties &prop, Attribute attr, function_ref emitError) { @@ -1209,6 +1254,7 @@ setPropertiesFromAttribute(VersionedProperties &prop, Attribute attr, prop.value2 = value2Attr.getValue().getSExtValue(); return success(); } + static DictionaryAttr getPropertiesAsAttribute(MLIRContext *ctx, const VersionedProperties &prop) { SmallVector attrs; @@ -1217,13 +1263,16 @@ getPropertiesAsAttribute(MLIRContext *ctx, const VersionedProperties &prop) { attrs.push_back(b.getNamedAttr("value2", b.getI32IntegerAttr(prop.value2))); return b.getDictionaryAttr(attrs); } + static llvm::hash_code computeHash(const VersionedProperties &prop) { return llvm::hash_combine(prop.value1, prop.value2); } + static void customPrintProperties(OpAsmPrinter &p, const VersionedProperties &prop) { p << prop.value1 << " | " << prop.value2; } + static ParseResult customParseProperties(OpAsmParser &parser, VersionedProperties &prop) { if (parser.parseInteger(prop.value1) || parser.parseVerticalBar() || @@ -1393,6 +1442,7 @@ void TestVersionedOpA::writeProperties(::mlir::DialectBytecodeWriter &writer) { prop.value2 = value2; return success(); } + void TestOpWithVersionedProperties::writeToMlirBytecode( ::mlir::DialectBytecodeWriter &writer, const test::VersionedProperties &prop) { diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td index 48b41d869876..61e181999ff2 100644 --- a/mlir/test/lib/Dialect/Test/TestOps.td +++ b/mlir/test/lib/Dialect/Test/TestOps.td @@ -2213,6 +2213,18 @@ def LoopBlockTerminatorOp : TEST_Op<"loop_block_term", }]; } +def TestNoTerminatorOp : TEST_Op<"switch_with_no_break", [ + NoTerminator, + DeclareOpInterfaceMethods + ]> { + let arguments = (ins Index:$arg, DenseI64ArrayAttr:$cases); + let regions = (region VariadicRegion>:$caseRegions); + + let assemblyFormat = [{ + $arg attr-dict custom($cases, $caseRegions) + }]; +} + //===----------------------------------------------------------------------===// // Test TableGen generated build() methods //===----------------------------------------------------------------------===// -- GitLab From 03ef103235752830da7b9ce5e825c0e3ddf7f45a Mon Sep 17 00:00:00 2001 From: Chris Cotter Date: Thu, 4 Jan 2024 15:47:14 -0500 Subject: [PATCH 039/728] [clang-tidy] Fix bug in modernize-use-emplace (#66169) emplace_back cannot construct an aggregate with arguments used to initialize the aggregate. Closes #62387 Test plan: Added test test from #62387 which contains code that should not be replaced by the check. --- .../clang-tidy/modernize/UseEmplaceCheck.cpp | 16 +++++++++++----- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++++ .../checkers/modernize/use-emplace.cpp | 13 +++++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp index 4438f0b22063..07e296cc2652 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp @@ -13,6 +13,10 @@ using namespace clang::ast_matchers; namespace clang::tidy::modernize { namespace { +AST_MATCHER_P(InitListExpr, initCountLeq, unsigned, N) { + return Node.getNumInits() <= N; +} + // Identical to hasAnyName, except it does not take template specifiers into // account. This is used to match the functions names as in // DefaultEmplacyFunctions below without caring about the template types of the @@ -205,11 +209,13 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) { auto HasConstructExpr = has(ignoringImplicit(SoughtConstructExpr)); // allow for T{} to be replaced, even if no CTOR is declared - auto HasConstructInitListExpr = has(initListExpr(anyOf( - allOf(has(SoughtConstructExpr), - has(cxxConstructExpr(argumentCountIs(0)))), - has(cxxBindTemporaryExpr(has(SoughtConstructExpr), - has(cxxConstructExpr(argumentCountIs(0)))))))); + auto HasConstructInitListExpr = + has(initListExpr(initCountLeq(1), + anyOf(allOf(has(SoughtConstructExpr), + has(cxxConstructExpr(argumentCountIs(0)))), + has(cxxBindTemporaryExpr( + has(SoughtConstructExpr), + has(cxxConstructExpr(argumentCountIs(0)))))))); auto HasBracedInitListExpr = anyOf(has(cxxBindTemporaryExpr(HasConstructInitListExpr)), HasConstructInitListExpr); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 571808a51596..4d25e2ebe85f 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -394,6 +394,10 @@ Changes in existing checks false-positives when constructing the container with ``count`` copies of elements with value ``value``. +- Improved :doc:`modernize-use-emplace + ` to not replace aggregates that + ``emplace`` cannot construct with aggregate initialization. + - Improved :doc:`modernize-use-equals-delete ` check to ignore false-positives when special member function is actually used or implicit. diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp index fead2b6151d0..f7b1ad55f5df 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp @@ -1183,6 +1183,11 @@ struct NonTrivialWithVector { std::vector it; }; +struct NonTrivialWithIntAndVector { + int x; + std::vector it; +}; + struct NonTrivialWithCtor { NonTrivialWithCtor(); NonTrivialWithCtor(std::vector const&); @@ -1332,6 +1337,14 @@ void testBracedInitTemporaries() { v3.push_back(NonTrivialWithCtor{{}}); v3.push_back({{0}}); v3.push_back({{}}); + + std::vector v4; + + // These should not be noticed or fixed; after the correction, the code won't + // compile. + v4.push_back(NonTrivialWithIntAndVector{1, {}}); + v4.push_back(NonTrivialWithIntAndVector{}); + v4.push_back({}); } void testWithPointerTypes() { -- GitLab From d67c2d85548437ef8c3bb12a29ea330180b87913 Mon Sep 17 00:00:00 2001 From: Piotr Zegar Date: Thu, 4 Jan 2024 18:20:40 +0000 Subject: [PATCH 040/728] [clang-tidy][NFC] Code format - leftover after #66169 Manually formating code via clang-format after previous commit merge. --- clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp index 07e296cc2652..430455a38f39 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp @@ -209,9 +209,8 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) { auto HasConstructExpr = has(ignoringImplicit(SoughtConstructExpr)); // allow for T{} to be replaced, even if no CTOR is declared - auto HasConstructInitListExpr = - has(initListExpr(initCountLeq(1), - anyOf(allOf(has(SoughtConstructExpr), + auto HasConstructInitListExpr = has(initListExpr( + initCountLeq(1), anyOf(allOf(has(SoughtConstructExpr), has(cxxConstructExpr(argumentCountIs(0)))), has(cxxBindTemporaryExpr( has(SoughtConstructExpr), -- GitLab From 2bc994456c5be2ab6d98b94de2349302577a9823 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Thu, 4 Jan 2024 15:51:14 -0500 Subject: [PATCH 041/728] [libc] major refactor of startup library (#76092) * separate initialization routines into _start and do_start for all architectures. * lift do_start as a separate object library to avoid code duplication. * (addtionally) address the problem of building hermetic libc with -fstack-pointer-* The `crt1.o` is now a merged result of three components: ``` ___ |___ x86_64 | |_______ start.cpp.o <- _start (loads process initial stack and aligns stack pointer) | |_______ tls.cpp.o <- init_tls, cleanup_tls, set_thread_pointer (TLS related routines) |___ do_start.cpp.o <- do_start (sets up global variables and invokes the main function) ``` --- libc/cmake/modules/LLVMLibCTestRules.cmake | 6 + libc/config/linux/app.h | 3 + libc/startup/linux/CMakeLists.txt | 25 ++- libc/startup/linux/aarch64/CMakeLists.txt | 19 +- libc/startup/linux/aarch64/start.cpp | 215 +------------------ libc/startup/linux/aarch64/tls.cpp | 86 ++++++++ libc/startup/linux/do_start.cpp | 140 ++++++++++++ libc/startup/linux/do_start.h | 14 ++ libc/startup/linux/riscv/CMakeLists.txt | 20 +- libc/startup/linux/riscv/start.cpp | 225 +------------------ libc/startup/linux/riscv/tls.cpp | 74 +++++++ libc/startup/linux/x86_64/CMakeLists.txt | 26 ++- libc/startup/linux/x86_64/start.cpp | 237 ++------------------- libc/startup/linux/x86_64/tls.cpp | 93 ++++++++ 14 files changed, 512 insertions(+), 671 deletions(-) create mode 100644 libc/startup/linux/aarch64/tls.cpp create mode 100644 libc/startup/linux/do_start.cpp create mode 100644 libc/startup/linux/do_start.h create mode 100644 libc/startup/linux/riscv/tls.cpp create mode 100644 libc/startup/linux/x86_64/tls.cpp diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake index 51d484b875ae..b69839afebf8 100644 --- a/libc/cmake/modules/LLVMLibCTestRules.cmake +++ b/libc/cmake/modules/LLVMLibCTestRules.cmake @@ -498,6 +498,9 @@ function(add_integration_test test_name) libc.src.string.memcpy libc.src.string.memmove libc.src.string.memset + # __stack_chk_fail should always be included to allow building libc with + # stack protector. + libc.src.compiler.__stack_chk_fail ) list(REMOVE_DUPLICATES fq_deps_list) @@ -665,6 +668,9 @@ function(add_libc_hermetic_test test_name) libc.src.string.memmove libc.src.string.memset libc.src.__support.StringUtil.error_to_string + # __stack_chk_fail should always be included to allow building libc with + # stack protector. + libc.src.compiler.__stack_chk_fail ) if(TARGET libc.src.time.clock) diff --git a/libc/config/linux/app.h b/libc/config/linux/app.h index 548c141fd705..1b3523deb1b2 100644 --- a/libc/config/linux/app.h +++ b/libc/config/linux/app.h @@ -119,6 +119,9 @@ void init_tls(TLSDescriptor &tls); // Cleanup the TLS area as described in |tls_descriptor|. void cleanup_tls(uintptr_t tls_addr, uintptr_t tls_size); +// Set the thread pointer for the current thread. +bool set_thread_ptr(uintptr_t val); + } // namespace LIBC_NAMESPACE #endif // LLVM_LIBC_CONFIG_LINUX_APP_H diff --git a/libc/startup/linux/CMakeLists.txt b/libc/startup/linux/CMakeLists.txt index 2d55a3656697..39bcca9cdba9 100644 --- a/libc/startup/linux/CMakeLists.txt +++ b/libc/startup/linux/CMakeLists.txt @@ -84,10 +84,33 @@ endif() add_subdirectory(${LIBC_TARGET_ARCHITECTURE}) +add_object_library( + do_start + SRCS + do_start.cpp + HDRS + do_start.h + DEPENDS + libc.config.linux.app_h + libc.include.sys_mman + libc.include.sys_syscall + libc.src.__support.threads.thread + libc.src.__support.OSUtil.osutil + libc.src.stdlib.exit + libc.src.stdlib.atexit + libc.src.unistd.environ + COMPILE_OPTIONS + -ffreestanding # To avoid compiler warnings about calling the main function. + -fno-builtin # avoid emit unexpected calls + -fno-stack-protector # stack protect canary is not available yet. +) + # TODO: factor out crt1 into multiple objects merge_relocatable_object( crt1 - .${LIBC_TARGET_ARCHITECTURE}.crt1 + .${LIBC_TARGET_ARCHITECTURE}.start + .${LIBC_TARGET_ARCHITECTURE}.tls + .do_start ) add_startup_object( diff --git a/libc/startup/linux/aarch64/CMakeLists.txt b/libc/startup/linux/aarch64/CMakeLists.txt index b47db8eb5d23..5ea6ae59abcb 100644 --- a/libc/startup/linux/aarch64/CMakeLists.txt +++ b/libc/startup/linux/aarch64/CMakeLists.txt @@ -1,17 +1,24 @@ add_startup_object( - crt1 + tls SRC - start.cpp + tls.cpp DEPENDS libc.config.linux.app_h libc.include.sys_mman libc.include.sys_syscall - libc.src.__support.threads.thread libc.src.__support.OSUtil.osutil - libc.src.stdlib.exit - libc.src.stdlib.atexit libc.src.string.memory_utils.inline_memcpy - libc.src.unistd.environ + COMPILE_OPTIONS + -fno-omit-frame-pointer + -ffreestanding # To avoid compiler warnings about calling the main function. +) + +add_startup_object( + start + SRC + start.cpp + DEPENDS + libc.config.linux.app_h COMPILE_OPTIONS -fno-omit-frame-pointer -ffreestanding # To avoid compiler warnings about calling the main function. diff --git a/libc/startup/linux/aarch64/start.cpp b/libc/startup/linux/aarch64/start.cpp index bc01582aeb49..d0a852687339 100644 --- a/libc/startup/linux/aarch64/start.cpp +++ b/libc/startup/linux/aarch64/start.cpp @@ -1,4 +1,4 @@ -//===-- Implementation of crt for aarch64 ---------------------------------===// +//===-- Implementation of _start for aarch64 ------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,213 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "config/linux/app.h" -#include "src/__support/OSUtil/syscall.h" -#include "src/__support/threads/thread.h" -#include "src/stdlib/atexit.h" -#include "src/stdlib/exit.h" -#include "src/string/memory_utils/inline_memcpy.h" - -#include - -#include -#include -#include -#include -#include -#include - -extern "C" int main(int, char **, char **); - -// Source documentation: -// https://github.com/ARM-software/abi-aa/tree/main/sysvabi64 - -namespace LIBC_NAMESPACE { - -#ifdef SYS_mmap2 -static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2; -#elif SYS_mmap -static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap; -#else -#error "mmap and mmap2 syscalls not available." -#endif - -AppProperties app; - -static ThreadAttributes main_thread_attrib; - -void init_tls(TLSDescriptor &tls_descriptor) { - if (app.tls.size == 0) { - tls_descriptor.size = 0; - tls_descriptor.tp = 0; - return; - } - - // aarch64 follows the variant 1 TLS layout: - // - // 1. First entry is the dynamic thread vector pointer - // 2. Second entry is a 8-byte reserved word. - // 3. Padding for alignment. - // 4. The TLS data from the ELF image. - // - // The thread pointer points to the first entry. - - const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t); - uintptr_t padding = 0; - const uintptr_t ALIGNMENT_MASK = app.tls.align - 1; - uintptr_t diff = size_of_pointers & ALIGNMENT_MASK; - if (diff != 0) - padding += (ALIGNMENT_MASK - diff) + 1; - - uintptr_t alloc_size = size_of_pointers + padding + app.tls.size; - - // We cannot call the mmap function here as the functions set errno on - // failure. Since errno is implemented via a thread local variable, we cannot - // use errno before TLS is setup. - long mmap_ret_val = LIBC_NAMESPACE::syscall_impl( - MMAP_SYSCALL_NUMBER, nullptr, alloc_size, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - // We cannot check the return value with MAP_FAILED as that is the return - // of the mmap function and not the mmap syscall. - if (mmap_ret_val < 0 && static_cast(mmap_ret_val) > -app.page_size) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - uintptr_t thread_ptr = uintptr_t(reinterpret_cast(mmap_ret_val)); - uintptr_t tls_addr = thread_ptr + size_of_pointers + padding; - LIBC_NAMESPACE::inline_memcpy(reinterpret_cast(tls_addr), - reinterpret_cast(app.tls.address), - app.tls.init_size); - tls_descriptor.size = alloc_size; - tls_descriptor.addr = thread_ptr; - tls_descriptor.tp = thread_ptr; -} - -void cleanup_tls(uintptr_t addr, uintptr_t size) { - if (size == 0) - return; - LIBC_NAMESPACE::syscall_impl(SYS_munmap, addr, size); -} - -static void set_thread_ptr(uintptr_t val) { __arm_wsr64("tpidr_el0", val); } - -using InitCallback = void(int, char **, char **); -using FiniCallback = void(void); - -extern "C" { -// These arrays are present in the .init_array and .fini_array sections. -// The symbols are inserted by linker when it sees references to them. -extern uintptr_t __preinit_array_start[]; -extern uintptr_t __preinit_array_end[]; -extern uintptr_t __init_array_start[]; -extern uintptr_t __init_array_end[]; -extern uintptr_t __fini_array_start[]; -extern uintptr_t __fini_array_end[]; -} - -static void call_init_array_callbacks(int argc, char **argv, char **env) { - size_t preinit_array_size = __preinit_array_end - __preinit_array_start; - for (size_t i = 0; i < preinit_array_size; ++i) - reinterpret_cast(__preinit_array_start[i])(argc, argv, env); - size_t init_array_size = __init_array_end - __init_array_start; - for (size_t i = 0; i < init_array_size; ++i) - reinterpret_cast(__init_array_start[i])(argc, argv, env); -} - -static void call_fini_array_callbacks() { - size_t fini_array_size = __fini_array_end - __fini_array_start; - for (size_t i = fini_array_size; i > 0; --i) - reinterpret_cast(__fini_array_start[i - 1])(); -} - -} // namespace LIBC_NAMESPACE - -using LIBC_NAMESPACE::app; -using LIBC_NAMESPACE::AuxEntry; - -__attribute__((noinline)) static void do_start() { - auto tid = LIBC_NAMESPACE::syscall_impl(SYS_gettid); - if (tid <= 0) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - LIBC_NAMESPACE::main_thread_attrib.tid = static_cast(tid); - - // After the argv array, is a 8-byte long NULL value before the array of env - // values. The end of the env values is marked by another 8-byte long NULL - // value. We step over it (the "+ 1" below) to get to the env values. - uint64_t *env_ptr = app.args->argv + app.args->argc + 1; - uint64_t *env_end_marker = env_ptr; - app.env_ptr = env_ptr; - while (*env_end_marker) - ++env_end_marker; - - // Initialize the POSIX global declared in unistd.h - environ = reinterpret_cast(env_ptr); - - // After the env array, is the aux-vector. The end of the aux-vector is - // denoted by an AT_NULL entry. - Elf64_Phdr *program_hdr_table = nullptr; - uintptr_t program_hdr_count; - app.auxv_ptr = reinterpret_cast(env_end_marker + 1); - for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) { - switch (aux_entry->id) { - case AT_PHDR: - program_hdr_table = reinterpret_cast(aux_entry->value); - break; - case AT_PHNUM: - program_hdr_count = aux_entry->value; - break; - case AT_PAGESZ: - app.page_size = aux_entry->value; - break; - default: - break; // TODO: Read other useful entries from the aux vector. - } - } - - app.tls.size = 0; - for (uintptr_t i = 0; i < program_hdr_count; ++i) { - Elf64_Phdr *phdr = program_hdr_table + i; - if (phdr->p_type != PT_TLS) - continue; - // TODO: p_vaddr value has to be adjusted for static-pie executables. - app.tls.address = phdr->p_vaddr; - app.tls.size = phdr->p_memsz; - app.tls.init_size = phdr->p_filesz; - app.tls.align = phdr->p_align; - } - - // This descriptor has to be static since its cleanup function cannot - // capture the context. - static LIBC_NAMESPACE::TLSDescriptor tls; - LIBC_NAMESPACE::init_tls(tls); - if (tls.size != 0) - LIBC_NAMESPACE::set_thread_ptr(tls.tp); - - LIBC_NAMESPACE::self.attrib = &LIBC_NAMESPACE::main_thread_attrib; - LIBC_NAMESPACE::main_thread_attrib.atexit_callback_mgr = - LIBC_NAMESPACE::internal::get_thread_atexit_callback_mgr(); - // We register the cleanup_tls function to be the last atexit callback to be - // invoked. It will tear down the TLS. Other callbacks may depend on TLS (such - // as the stack protector canary). - LIBC_NAMESPACE::atexit( - []() { LIBC_NAMESPACE::cleanup_tls(tls.tp, tls.size); }); - // We want the fini array callbacks to be run after other atexit - // callbacks are run. So, we register them before running the init - // array callbacks as they can potentially register their own atexit - // callbacks. - LIBC_NAMESPACE::atexit(&LIBC_NAMESPACE::call_fini_array_callbacks); - - LIBC_NAMESPACE::call_init_array_callbacks( - static_cast(app.args->argc), - reinterpret_cast(app.args->argv), - reinterpret_cast(env_ptr)); - - int retval = main(static_cast(app.args->argc), - reinterpret_cast(app.args->argv), - reinterpret_cast(env_ptr)); - - LIBC_NAMESPACE::exit(retval); -} - -extern "C" void _start() { +#include "startup/linux/do_start.h" +extern "C" [[noreturn]] void _start() { // Skip the Frame Pointer and the Link Register // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst // Section 6.2.3. Note that this only works if the current function @@ -223,7 +18,7 @@ extern "C" void _start() { // will take us to the previous stack pointer. That is the reason why the // actual business logic of the startup code is pushed into a non-inline // function do_start so that this function is free of any stack usage. - app.args = reinterpret_cast( + LIBC_NAMESPACE::app.args = reinterpret_cast( reinterpret_cast(__builtin_frame_address(0)) + 2); - do_start(); + LIBC_NAMESPACE::do_start(); } diff --git a/libc/startup/linux/aarch64/tls.cpp b/libc/startup/linux/aarch64/tls.cpp new file mode 100644 index 000000000000..f2579e821b1b --- /dev/null +++ b/libc/startup/linux/aarch64/tls.cpp @@ -0,0 +1,86 @@ +//===-- Implementation of tls for aarch64 ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/threads/thread.h" +#include "src/string/memory_utils/inline_memcpy.h" +#include "startup/linux/do_start.h" + +#include +#include +#include + +// Source documentation: +// https://github.com/ARM-software/abi-aa/tree/main/sysvabi64 + +namespace LIBC_NAMESPACE { + +#ifdef SYS_mmap2 +static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2; +#elif SYS_mmap +static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap; +#else +#error "mmap and mmap2 syscalls not available." +#endif + +void init_tls(TLSDescriptor &tls_descriptor) { + if (app.tls.size == 0) { + tls_descriptor.size = 0; + tls_descriptor.tp = 0; + return; + } + + // aarch64 follows the variant 1 TLS layout: + // + // 1. First entry is the dynamic thread vector pointer + // 2. Second entry is a 8-byte reserved word. + // 3. Padding for alignment. + // 4. The TLS data from the ELF image. + // + // The thread pointer points to the first entry. + + const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t); + uintptr_t padding = 0; + const uintptr_t ALIGNMENT_MASK = app.tls.align - 1; + uintptr_t diff = size_of_pointers & ALIGNMENT_MASK; + if (diff != 0) + padding += (ALIGNMENT_MASK - diff) + 1; + + uintptr_t alloc_size = size_of_pointers + padding + app.tls.size; + + // We cannot call the mmap function here as the functions set errno on + // failure. Since errno is implemented via a thread local variable, we cannot + // use errno before TLS is setup. + long mmap_ret_val = syscall_impl(MMAP_SYSCALL_NUMBER, nullptr, + alloc_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + // We cannot check the return value with MAP_FAILED as that is the return + // of the mmap function and not the mmap syscall. + if (mmap_ret_val < 0 && static_cast(mmap_ret_val) > -app.page_size) + syscall_impl(SYS_exit, 1); + uintptr_t thread_ptr = uintptr_t(reinterpret_cast(mmap_ret_val)); + uintptr_t tls_addr = thread_ptr + size_of_pointers + padding; + inline_memcpy(reinterpret_cast(tls_addr), + reinterpret_cast(app.tls.address), + app.tls.init_size); + tls_descriptor.size = alloc_size; + tls_descriptor.addr = thread_ptr; + tls_descriptor.tp = thread_ptr; +} + +void cleanup_tls(uintptr_t addr, uintptr_t size) { + if (size == 0) + return; + syscall_impl(SYS_munmap, addr, size); +} + +bool set_thread_ptr(uintptr_t val) { + __arm_wsr64("tpidr_el0", val); + return true; +} +} // namespace LIBC_NAMESPACE diff --git a/libc/startup/linux/do_start.cpp b/libc/startup/linux/do_start.cpp new file mode 100644 index 000000000000..05dbd4488f58 --- /dev/null +++ b/libc/startup/linux/do_start.cpp @@ -0,0 +1,140 @@ +//===-- Implementation file of do_start -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "startup/linux/do_start.h" +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/threads/thread.h" +#include "src/stdlib/atexit.h" +#include "src/stdlib/exit.h" +#include "src/unistd/environ.h" + +#include +#include +#include +#include +#include + +extern "C" int main(int argc, char **argv, char **envp); + +extern "C" { +// These arrays are present in the .init_array and .fini_array sections. +// The symbols are inserted by linker when it sees references to them. +extern uintptr_t __preinit_array_start[]; +extern uintptr_t __preinit_array_end[]; +extern uintptr_t __init_array_start[]; +extern uintptr_t __init_array_end[]; +extern uintptr_t __fini_array_start[]; +extern uintptr_t __fini_array_end[]; +} + +namespace LIBC_NAMESPACE { +// TODO: this symbol will be moved to config.linux.app +AppProperties app; + +using InitCallback = void(int, char **, char **); +using FiniCallback = void(void); + +static void call_init_array_callbacks(int argc, char **argv, char **env) { + size_t preinit_array_size = __preinit_array_end - __preinit_array_start; + for (size_t i = 0; i < preinit_array_size; ++i) + reinterpret_cast(__preinit_array_start[i])(argc, argv, env); + size_t init_array_size = __init_array_end - __init_array_start; + for (size_t i = 0; i < init_array_size; ++i) + reinterpret_cast(__init_array_start[i])(argc, argv, env); +} + +static void call_fini_array_callbacks() { + size_t fini_array_size = __fini_array_end - __fini_array_start; + for (size_t i = fini_array_size; i > 0; --i) + reinterpret_cast(__fini_array_start[i - 1])(); +} + +static ThreadAttributes main_thread_attrib; + +[[noreturn]] void do_start() { + auto tid = syscall_impl(SYS_gettid); + if (tid <= 0) + syscall_impl(SYS_exit, 1); + main_thread_attrib.tid = static_cast(tid); + + // After the argv array, is a 8-byte long NULL value before the array of env + // values. The end of the env values is marked by another 8-byte long NULL + // value. We step over it (the "+ 1" below) to get to the env values. + ArgVEntryType *env_ptr = app.args->argv + app.args->argc + 1; + ArgVEntryType *env_end_marker = env_ptr; + app.env_ptr = env_ptr; + while (*env_end_marker) + ++env_end_marker; + + // Initialize the POSIX global declared in unistd.h + environ = reinterpret_cast(env_ptr); + + // After the env array, is the aux-vector. The end of the aux-vector is + // denoted by an AT_NULL entry. + Elf64_Phdr *program_hdr_table = nullptr; + uintptr_t program_hdr_count = 0; + app.auxv_ptr = reinterpret_cast(env_end_marker + 1); + for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) { + switch (aux_entry->id) { + case AT_PHDR: + program_hdr_table = reinterpret_cast(aux_entry->value); + break; + case AT_PHNUM: + program_hdr_count = aux_entry->value; + break; + case AT_PAGESZ: + app.page_size = aux_entry->value; + break; + default: + break; // TODO: Read other useful entries from the aux vector. + } + } + + app.tls.size = 0; + for (uintptr_t i = 0; i < program_hdr_count; ++i) { + Elf64_Phdr *phdr = program_hdr_table + i; + if (phdr->p_type != PT_TLS) + continue; + // TODO: p_vaddr value has to be adjusted for static-pie executables. + app.tls.address = phdr->p_vaddr; + app.tls.size = phdr->p_memsz; + app.tls.init_size = phdr->p_filesz; + app.tls.align = phdr->p_align; + } + + // This descriptor has to be static since its cleanup function cannot + // capture the context. + static TLSDescriptor tls; + init_tls(tls); + if (tls.size != 0 && !set_thread_ptr(tls.tp)) + syscall_impl(SYS_exit, 1); + + self.attrib = &main_thread_attrib; + main_thread_attrib.atexit_callback_mgr = + internal::get_thread_atexit_callback_mgr(); + // We register the cleanup_tls function to be the last atexit callback to be + // invoked. It will tear down the TLS. Other callbacks may depend on TLS (such + // as the stack protector canary). + atexit([]() { cleanup_tls(tls.tp, tls.size); }); + // We want the fini array callbacks to be run after other atexit + // callbacks are run. So, we register them before running the init + // array callbacks as they can potentially register their own atexit + // callbacks. + atexit(&call_fini_array_callbacks); + + call_init_array_callbacks(static_cast(app.args->argc), + reinterpret_cast(app.args->argv), + reinterpret_cast(env_ptr)); + + int retval = main(static_cast(app.args->argc), + reinterpret_cast(app.args->argv), + reinterpret_cast(env_ptr)); + + exit(retval); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/startup/linux/do_start.h b/libc/startup/linux/do_start.h new file mode 100644 index 000000000000..a0e7a3cd6956 --- /dev/null +++ b/libc/startup/linux/do_start.h @@ -0,0 +1,14 @@ +//===-- Header file of do_start -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "config/linux/app.h" + +namespace LIBC_NAMESPACE { +// setup the libc runtime and invoke the main routine. +[[noreturn]] void do_start(); +} // namespace LIBC_NAMESPACE diff --git a/libc/startup/linux/riscv/CMakeLists.txt b/libc/startup/linux/riscv/CMakeLists.txt index b47db8eb5d23..3717784233c1 100644 --- a/libc/startup/linux/riscv/CMakeLists.txt +++ b/libc/startup/linux/riscv/CMakeLists.txt @@ -1,17 +1,25 @@ add_startup_object( - crt1 + tls SRC - start.cpp + tls.cpp DEPENDS libc.config.linux.app_h libc.include.sys_mman libc.include.sys_syscall - libc.src.__support.threads.thread libc.src.__support.OSUtil.osutil - libc.src.stdlib.exit - libc.src.stdlib.atexit libc.src.string.memory_utils.inline_memcpy - libc.src.unistd.environ + COMPILE_OPTIONS + -fno-omit-frame-pointer + -ffreestanding # To avoid compiler warnings about calling the main function. +) + +add_startup_object( + start + SRC + start.cpp + DEPENDS + libc.config.linux.app_h + libc.src.__support.macros.attributes COMPILE_OPTIONS -fno-omit-frame-pointer -ffreestanding # To avoid compiler warnings about calling the main function. diff --git a/libc/startup/linux/riscv/start.cpp b/libc/startup/linux/riscv/start.cpp index 5b6e5bde8da8..389f71a66d30 100644 --- a/libc/startup/linux/riscv/start.cpp +++ b/libc/startup/linux/riscv/start.cpp @@ -1,223 +1,20 @@ -//===-- Implementation of crt for riscv64 ---------------------------------===// +//===-- Implementation of _start for riscv --------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -#include "config/linux/app.h" -#include "src/__support/OSUtil/syscall.h" -#include "src/__support/threads/thread.h" -#include "src/stdlib/atexit.h" -#include "src/stdlib/exit.h" -#include "src/string/memory_utils/inline_memcpy.h" - -#include -#include -#include -#include -#include -#include - -extern "C" int main(int, char **, char **); - -namespace LIBC_NAMESPACE { - -#ifdef SYS_mmap2 -static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2; -#elif SYS_mmap -static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap; -#else -#error "mmap and mmap2 syscalls not available." -#endif - -AppProperties app; - -static ThreadAttributes main_thread_attrib; - -void init_tls(TLSDescriptor &tls_descriptor) { - if (app.tls.size == 0) { - tls_descriptor.size = 0; - tls_descriptor.tp = 0; - return; - } - - // riscv64 follows the variant 1 TLS layout: - const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t); - uintptr_t padding = 0; - const uintptr_t ALIGNMENT_MASK = app.tls.align - 1; - uintptr_t diff = size_of_pointers & ALIGNMENT_MASK; - if (diff != 0) - padding += (ALIGNMENT_MASK - diff) + 1; - - uintptr_t alloc_size = size_of_pointers + padding + app.tls.size; - - // We cannot call the mmap function here as the functions set errno on - // failure. Since errno is implemented via a thread local variable, we cannot - // use errno before TLS is setup. - long mmap_ret_val = LIBC_NAMESPACE::syscall_impl( - MMAP_SYSCALL_NUMBER, nullptr, alloc_size, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - // We cannot check the return value with MAP_FAILED as that is the return - // of the mmap function and not the mmap syscall. - if (mmap_ret_val < 0 && static_cast(mmap_ret_val) > -app.page_size) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - uintptr_t thread_ptr = uintptr_t(reinterpret_cast(mmap_ret_val)); - uintptr_t tls_addr = thread_ptr + size_of_pointers + padding; - LIBC_NAMESPACE::inline_memcpy(reinterpret_cast(tls_addr), - reinterpret_cast(app.tls.address), - app.tls.init_size); - tls_descriptor.size = alloc_size; - tls_descriptor.addr = thread_ptr; - tls_descriptor.tp = tls_addr; -} - -void cleanup_tls(uintptr_t addr, uintptr_t size) { - if (size == 0) - return; - LIBC_NAMESPACE::syscall_impl(SYS_munmap, addr, size); -} - -static void set_thread_ptr(uintptr_t val) { - LIBC_INLINE_ASM("mv tp, %0\n\t" : : "r"(val)); -} - -using InitCallback = void(int, char **, char **); -using FiniCallback = void(void); - -extern "C" { -// These arrays are present in the .init_array and .fini_array sections. -// The symbols are inserted by linker when it sees references to them. -extern uintptr_t __preinit_array_start[]; -extern uintptr_t __preinit_array_end[]; -extern uintptr_t __init_array_start[]; -extern uintptr_t __init_array_end[]; -extern uintptr_t __fini_array_start[]; -extern uintptr_t __fini_array_end[]; -} - -static void call_init_array_callbacks(int argc, char **argv, char **env) { - size_t preinit_array_size = __preinit_array_end - __preinit_array_start; - for (size_t i = 0; i < preinit_array_size; ++i) - reinterpret_cast(__preinit_array_start[i])(argc, argv, env); - size_t init_array_size = __init_array_end - __init_array_start; - for (size_t i = 0; i < init_array_size; ++i) - reinterpret_cast(__init_array_start[i])(argc, argv, env); -} - -static void call_fini_array_callbacks() { - size_t fini_array_size = __fini_array_end - __fini_array_start; - for (size_t i = fini_array_size; i > 0; --i) - reinterpret_cast(__fini_array_start[i - 1])(); -} - -} // namespace LIBC_NAMESPACE - -using LIBC_NAMESPACE::app; -using LIBC_NAMESPACE::AuxEntry; - -#if defined(LIBC_TARGET_ARCH_IS_X86_64) || \ - defined(LIBC_TARGET_ARCH_IS_AARCH64) || \ - defined(LIBC_TARGET_ARCH_IS_RISCV64) -typedef Elf64_Phdr PgrHdrTableType; -#elif defined(LIBC_TARGET_ARCH_IS_RISCV32) -typedef Elf32_Phdr PgrHdrTableType; -#else -#error "Program header table type is not defined for the target platform." -#endif - -__attribute__((noinline)) static void do_start() { - LIBC_INLINE_ASM(".option push\n\t" - ".option norelax\n\t" - "lla gp, __global_pointer$\n\t" - ".option pop\n\t"); - auto tid = LIBC_NAMESPACE::syscall_impl(SYS_gettid); - if (tid <= 0) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - LIBC_NAMESPACE::main_thread_attrib.tid = static_cast(tid); - - // After the argv array, is a 8-byte long NULL value before the array of env - // values. The end of the env values is marked by another 8-byte long NULL - // value. We step over it (the "+ 1" below) to get to the env values. - LIBC_NAMESPACE::ArgVEntryType *env_ptr = app.args->argv + app.args->argc + 1; - LIBC_NAMESPACE::ArgVEntryType *env_end_marker = env_ptr; - app.env_ptr = env_ptr; - while (*env_end_marker) - ++env_end_marker; - - // Initialize the POSIX global declared in unistd.h - environ = reinterpret_cast(env_ptr); - - // After the env array, is the aux-vector. The end of the aux-vector is - // denoted by an AT_NULL entry. - PgrHdrTableType *program_hdr_table = nullptr; - uintptr_t program_hdr_count; - app.auxv_ptr = reinterpret_cast(env_end_marker + 1); - for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) { - switch (aux_entry->id) { - case AT_PHDR: - program_hdr_table = reinterpret_cast(aux_entry->value); - break; - case AT_PHNUM: - program_hdr_count = aux_entry->value; - break; - case AT_PAGESZ: - app.page_size = aux_entry->value; - break; - default: - break; // TODO: Read other useful entries from the aux vector. - } - } - - app.tls.size = 0; - for (uintptr_t i = 0; i < program_hdr_count; ++i) { - PgrHdrTableType *phdr = program_hdr_table + i; - if (phdr->p_type != PT_TLS) - continue; - // TODO: p_vaddr value has to be adjusted for static-pie executables. - app.tls.address = phdr->p_vaddr; - app.tls.size = phdr->p_memsz; - app.tls.init_size = phdr->p_filesz; - app.tls.align = phdr->p_align; - } - - // This descriptor has to be static since its cleanup function cannot - // capture the context. - static LIBC_NAMESPACE::TLSDescriptor tls; - LIBC_NAMESPACE::init_tls(tls); - if (tls.size != 0) - LIBC_NAMESPACE::set_thread_ptr(tls.tp); - - LIBC_NAMESPACE::self.attrib = &LIBC_NAMESPACE::main_thread_attrib; - LIBC_NAMESPACE::main_thread_attrib.atexit_callback_mgr = - LIBC_NAMESPACE::internal::get_thread_atexit_callback_mgr(); - // We register the cleanup_tls function to be the last atexit callback to be - // invoked. It will tear down the TLS. Other callbacks may depend on TLS (such - // as the stack protector canary). - LIBC_NAMESPACE::atexit( - []() { LIBC_NAMESPACE::cleanup_tls(tls.tp, tls.size); }); - // We want the fini array callbacks to be run after other atexit - // callbacks are run. So, we register them before running the init - // array callbacks as they can potentially register their own atexit - // callbacks. - LIBC_NAMESPACE::atexit(&LIBC_NAMESPACE::call_fini_array_callbacks); - - LIBC_NAMESPACE::call_init_array_callbacks( - static_cast(app.args->argc), - reinterpret_cast(app.args->argv), - reinterpret_cast(env_ptr)); - - int retval = main(static_cast(app.args->argc), - reinterpret_cast(app.args->argv), - reinterpret_cast(env_ptr)); - - LIBC_NAMESPACE::exit(retval); -} - -extern "C" void _start() { +#include "src/__support/macros/attributes.h" +#include "startup/linux/do_start.h" + +extern "C" [[noreturn]] void _start() { + asm volatile(".option push\n\t" + ".option norelax\n\t" + "lla gp, __global_pointer$\n\t" + ".option pop\n\t"); // Fetch the args using the frame pointer. - app.args = reinterpret_cast( + LIBC_NAMESPACE::app.args = reinterpret_cast( reinterpret_cast(__builtin_frame_address(0))); - do_start(); + LIBC_NAMESPACE::do_start(); } diff --git a/libc/startup/linux/riscv/tls.cpp b/libc/startup/linux/riscv/tls.cpp new file mode 100644 index 000000000000..997912c77e73 --- /dev/null +++ b/libc/startup/linux/riscv/tls.cpp @@ -0,0 +1,74 @@ +//===-- Implementation of tls for riscv -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/threads/thread.h" +#include "src/string/memory_utils/inline_memcpy.h" +#include "startup/linux/do_start.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +#ifdef SYS_mmap2 +static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2; +#elif SYS_mmap +static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap; +#else +#error "mmap and mmap2 syscalls not available." +#endif + +void init_tls(TLSDescriptor &tls_descriptor) { + if (app.tls.size == 0) { + tls_descriptor.size = 0; + tls_descriptor.tp = 0; + return; + } + + // riscv64 follows the variant 1 TLS layout: + const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t); + uintptr_t padding = 0; + const uintptr_t ALIGNMENT_MASK = app.tls.align - 1; + uintptr_t diff = size_of_pointers & ALIGNMENT_MASK; + if (diff != 0) + padding += (ALIGNMENT_MASK - diff) + 1; + + uintptr_t alloc_size = size_of_pointers + padding + app.tls.size; + + // We cannot call the mmap function here as the functions set errno on + // failure. Since errno is implemented via a thread local variable, we cannot + // use errno before TLS is setup. + long mmap_ret_val = syscall_impl(MMAP_SYSCALL_NUMBER, nullptr, + alloc_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + // We cannot check the return value with MAP_FAILED as that is the return + // of the mmap function and not the mmap syscall. + if (mmap_ret_val < 0 && static_cast(mmap_ret_val) > -app.page_size) + syscall_impl(SYS_exit, 1); + uintptr_t thread_ptr = uintptr_t(reinterpret_cast(mmap_ret_val)); + uintptr_t tls_addr = thread_ptr + size_of_pointers + padding; + inline_memcpy(reinterpret_cast(tls_addr), + reinterpret_cast(app.tls.address), + app.tls.init_size); + tls_descriptor.size = alloc_size; + tls_descriptor.addr = thread_ptr; + tls_descriptor.tp = tls_addr; +} + +void cleanup_tls(uintptr_t addr, uintptr_t size) { + if (size == 0) + return; + syscall_impl(SYS_munmap, addr, size); +} + +bool set_thread_ptr(uintptr_t val) { + LIBC_INLINE_ASM("mv tp, %0\n\t" : : "r"(val)); + return true; +} +} // namespace LIBC_NAMESPACE diff --git a/libc/startup/linux/x86_64/CMakeLists.txt b/libc/startup/linux/x86_64/CMakeLists.txt index aac5a0626a17..30da7ab4e1ec 100644 --- a/libc/startup/linux/x86_64/CMakeLists.txt +++ b/libc/startup/linux/x86_64/CMakeLists.txt @@ -1,22 +1,30 @@ add_startup_object( - crt1 + tls SRC - start.cpp + tls.cpp DEPENDS libc.config.linux.app_h libc.include.sys_mman libc.include.sys_syscall - libc.include.unistd - libc.src.__support.threads.thread libc.src.__support.OSUtil.osutil - libc.src.stdlib.exit - libc.src.stdlib.abort - libc.src.stdlib.atexit libc.src.string.memory_utils.inline_memcpy - libc.src.unistd.environ COMPILE_OPTIONS -fno-stack-protector -fno-omit-frame-pointer - -ffreestanding # To avoid compiler warnings about calling the main function. + -ffreestanding + -fno-builtin +) + +add_startup_object( + start + SRC + start.cpp + DEPENDS + libc.config.linux.app_h + libc.src.__support.macros.attributes + COMPILE_OPTIONS + -fno-stack-protector + -fno-omit-frame-pointer + -ffreestanding -fno-builtin ) diff --git a/libc/startup/linux/x86_64/start.cpp b/libc/startup/linux/x86_64/start.cpp index bc03a3cb1de2..25da25a496da 100644 --- a/libc/startup/linux/x86_64/start.cpp +++ b/libc/startup/linux/x86_64/start.cpp @@ -1,151 +1,18 @@ -//===-- Implementation of crt for x86_64 ----------------------------------===// +//===-- Implementation of _start for x86_64 -------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -#include "config/linux/app.h" -#include "src/__support/OSUtil/io.h" -#include "src/__support/OSUtil/syscall.h" -#include "src/__support/threads/thread.h" -#include "src/stdlib/abort.h" -#include "src/stdlib/atexit.h" -#include "src/stdlib/exit.h" -#include "src/string/memory_utils/inline_memcpy.h" - -#include -#include -#include -#include -#include -#include -#include - -extern "C" int main(int, char **, char **); - -namespace LIBC_NAMESPACE { - -#ifdef SYS_mmap2 -static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2; -#elif SYS_mmap -static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap; -#else -#error "mmap and mmap2 syscalls not available." -#endif - -AppProperties app; - -static ThreadAttributes main_thread_attrib; - -// TODO: The function is x86_64 specific. Move it to config/linux/app.h -// and generalize it. Also, dynamic loading is not handled currently. -void init_tls(TLSDescriptor &tls_descriptor) { - if (app.tls.size == 0) { - tls_descriptor.size = 0; - tls_descriptor.tp = 0; - return; - } - - // We will assume the alignment is always a power of two. - uintptr_t tls_size = app.tls.size & -app.tls.align; - if (tls_size != app.tls.size) - tls_size += app.tls.align; - - // Per the x86_64 TLS ABI, the entry pointed to by the thread pointer is the - // address of the TLS block. So, we add more size to accomodate this address - // entry. - // We also need to include space for the stack canary. The canary is at - // offset 0x28 (40) and is of size uintptr_t. - uintptr_t tls_size_with_addr = tls_size + sizeof(uintptr_t) + 40; - - // We cannot call the mmap function here as the functions set errno on - // failure. Since errno is implemented via a thread local variable, we cannot - // use errno before TLS is setup. - long mmap_retval = LIBC_NAMESPACE::syscall_impl( - MMAP_SYSCALL_NUMBER, nullptr, tls_size_with_addr, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - // We cannot check the return value with MAP_FAILED as that is the return - // of the mmap function and not the mmap syscall. - if (mmap_retval < 0 && static_cast(mmap_retval) > -app.page_size) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - uintptr_t *tls_addr = reinterpret_cast(mmap_retval); - - // x86_64 TLS faces down from the thread pointer with the first entry - // pointing to the address of the first real TLS byte. - uintptr_t end_ptr = reinterpret_cast(tls_addr) + tls_size; - *reinterpret_cast(end_ptr) = end_ptr; - - LIBC_NAMESPACE::inline_memcpy(reinterpret_cast(tls_addr), - reinterpret_cast(app.tls.address), - app.tls.init_size); - uintptr_t *stack_guard_addr = reinterpret_cast(end_ptr + 40); - // Setting the stack guard to a random value. - // We cannot call the get_random function here as the function sets errno on - // failure. Since errno is implemented via a thread local variable, we cannot - // use errno before TLS is setup. - ssize_t stack_guard_retval = LIBC_NAMESPACE::syscall_impl( - SYS_getrandom, reinterpret_cast(stack_guard_addr), sizeof(uint64_t), - 0); - if (stack_guard_retval < 0) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - - tls_descriptor = {tls_size_with_addr, reinterpret_cast(tls_addr), - end_ptr}; - return; -} - -void cleanup_tls(uintptr_t addr, uintptr_t size) { - if (size == 0) - return; - LIBC_NAMESPACE::syscall_impl(SYS_munmap, addr, size); -} - -// Sets the thread pointer to |val|. Returns true on success, false on failure. -static bool set_thread_ptr(uintptr_t val) { - return LIBC_NAMESPACE::syscall_impl(SYS_arch_prctl, ARCH_SET_FS, val) != -1; -} - -using InitCallback = void(int, char **, char **); -using FiniCallback = void(void); - -extern "C" { -// These arrays are present in the .init_array and .fini_array sections. -// The symbols are inserted by linker when it sees references to them. -extern uintptr_t __preinit_array_start[]; -extern uintptr_t __preinit_array_end[]; -extern uintptr_t __init_array_start[]; -extern uintptr_t __init_array_end[]; -extern uintptr_t __fini_array_start[]; -extern uintptr_t __fini_array_end[]; -} - -static void call_init_array_callbacks(int argc, char **argv, char **env) { - size_t preinit_array_size = __preinit_array_end - __preinit_array_start; - for (size_t i = 0; i < preinit_array_size; ++i) - reinterpret_cast(__preinit_array_start[i])(argc, argv, env); - size_t init_array_size = __init_array_end - __init_array_start; - for (size_t i = 0; i < init_array_size; ++i) - reinterpret_cast(__init_array_start[i])(argc, argv, env); -} - -static void call_fini_array_callbacks() { - size_t fini_array_size = __fini_array_end - __fini_array_start; - for (size_t i = fini_array_size; i > 0; --i) - reinterpret_cast(__fini_array_start[i - 1])(); -} - -} // namespace LIBC_NAMESPACE - -using LIBC_NAMESPACE::app; -using LIBC_NAMESPACE::AuxEntry; - -extern "C" void _start() { - // This TU is compiled with -fno-omit-frame-pointer. Hence, the previous value - // of the base pointer is pushed on to the stack. So, we step over it (the - // "+ 1" below) to get to the args. - app.args = reinterpret_cast( +#include "src/__support/macros/attributes.h" +#include "startup/linux/do_start.h" + +extern "C" [[noreturn]] void _start() { + // This TU is compiled with -fno-omit-frame-pointer. Hence, the previous + // value of the base pointer is pushed on to the stack. So, we step over + // it (the "+ 1" below) to get to the args. + LIBC_NAMESPACE::app.args = reinterpret_cast( reinterpret_cast(__builtin_frame_address(0)) + 1); // The x86_64 ABI requires that the stack pointer is aligned to a 16-byte @@ -159,88 +26,8 @@ extern "C" void _start() { // compilers can generate code assuming the alignment as required by the ABI. // If the stack pointers as setup by the OS are already aligned, then the // following code is a NOP. - __asm__ __volatile__("andq $0xfffffffffffffff0, %rsp\n\t"); - __asm__ __volatile__("andq $0xfffffffffffffff0, %rbp\n\t"); - - auto tid = LIBC_NAMESPACE::syscall_impl(SYS_gettid); - if (tid <= 0) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - LIBC_NAMESPACE::main_thread_attrib.tid = static_cast(tid); - - // After the argv array, is a 8-byte long NULL value before the array of env - // values. The end of the env values is marked by another 8-byte long NULL - // value. We step over it (the "+ 1" below) to get to the env values. - uint64_t *env_ptr = app.args->argv + app.args->argc + 1; - uint64_t *env_end_marker = env_ptr; - app.env_ptr = env_ptr; - while (*env_end_marker) - ++env_end_marker; - - // Initialize the POSIX global declared in unistd.h - environ = reinterpret_cast(env_ptr); - - // After the env array, is the aux-vector. The end of the aux-vector is - // denoted by an AT_NULL entry. - Elf64_Phdr *program_hdr_table = nullptr; - uintptr_t program_hdr_count = 0; - app.auxv_ptr = reinterpret_cast(env_end_marker + 1); - for (auto *aux_entry = app.auxv_ptr; aux_entry->id != AT_NULL; ++aux_entry) { - switch (aux_entry->id) { - case AT_PHDR: - program_hdr_table = reinterpret_cast(aux_entry->value); - break; - case AT_PHNUM: - program_hdr_count = aux_entry->value; - break; - case AT_PAGESZ: - app.page_size = aux_entry->value; - break; - default: - break; // TODO: Read other useful entries from the aux vector. - } - } - - app.tls.size = 0; - for (uintptr_t i = 0; i < program_hdr_count; ++i) { - Elf64_Phdr *phdr = program_hdr_table + i; - if (phdr->p_type != PT_TLS) - continue; - // TODO: p_vaddr value has to be adjusted for static-pie executables. - app.tls.address = phdr->p_vaddr; - app.tls.size = phdr->p_memsz; - app.tls.init_size = phdr->p_filesz; - app.tls.align = phdr->p_align; - } - - // This descriptor has to be static since its cleanup function cannot - // capture the context. - static LIBC_NAMESPACE::TLSDescriptor tls; - LIBC_NAMESPACE::init_tls(tls); - if (tls.size != 0 && !LIBC_NAMESPACE::set_thread_ptr(tls.tp)) - LIBC_NAMESPACE::syscall_impl(SYS_exit, 1); - - LIBC_NAMESPACE::self.attrib = &LIBC_NAMESPACE::main_thread_attrib; - LIBC_NAMESPACE::main_thread_attrib.atexit_callback_mgr = - LIBC_NAMESPACE::internal::get_thread_atexit_callback_mgr(); - // We register the cleanup_tls function to be the last atexit callback to be - // invoked. It will tear down the TLS. Other callbacks may depend on TLS (such - // as the stack protector canary). - LIBC_NAMESPACE::atexit( - []() { LIBC_NAMESPACE::cleanup_tls(tls.tp, tls.size); }); - // We want the fini array callbacks to be run after other atexit - // callbacks are run. So, we register them before running the init - // array callbacks as they can potentially register their own atexit - // callbacks. - LIBC_NAMESPACE::atexit(&LIBC_NAMESPACE::call_fini_array_callbacks); - - LIBC_NAMESPACE::call_init_array_callbacks( - static_cast(app.args->argc), - reinterpret_cast(app.args->argv), - reinterpret_cast(env_ptr)); - - int retval = main(static_cast(app.args->argc), - reinterpret_cast(app.args->argv), - reinterpret_cast(env_ptr)); + asm volatile("andq $0xfffffffffffffff0, %rsp\n\t"); + asm volatile("andq $0xfffffffffffffff0, %rbp\n\t"); - LIBC_NAMESPACE::exit(retval); + LIBC_NAMESPACE::do_start(); } diff --git a/libc/startup/linux/x86_64/tls.cpp b/libc/startup/linux/x86_64/tls.cpp new file mode 100644 index 000000000000..8b0fa9873624 --- /dev/null +++ b/libc/startup/linux/x86_64/tls.cpp @@ -0,0 +1,93 @@ +//===-- Implementation of tls for x86_64 ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/OSUtil/syscall.h" +#include "src/string/memory_utils/inline_memcpy.h" +#include "startup/linux/do_start.h" + +#include +#include +#include + +namespace LIBC_NAMESPACE { + +#ifdef SYS_mmap2 +static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2; +#elif SYS_mmap +static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap; +#else +#error "mmap and mmap2 syscalls not available." +#endif + +// TODO: Also generalize this routine and handle dynamic loading properly. +void init_tls(TLSDescriptor &tls_descriptor) { + if (app.tls.size == 0) { + tls_descriptor.size = 0; + tls_descriptor.tp = 0; + return; + } + + // We will assume the alignment is always a power of two. + uintptr_t tls_size = app.tls.size & -app.tls.align; + if (tls_size != app.tls.size) + tls_size += app.tls.align; + + // Per the x86_64 TLS ABI, the entry pointed to by the thread pointer is the + // address of the TLS block. So, we add more size to accomodate this address + // entry. + // We also need to include space for the stack canary. The canary is at + // offset 0x28 (40) and is of size uintptr_t. + uintptr_t tls_size_with_addr = tls_size + sizeof(uintptr_t) + 40; + + // We cannot call the mmap function here as the functions set errno on + // failure. Since errno is implemented via a thread local variable, we cannot + // use errno before TLS is setup. + long mmap_retval = syscall_impl( + MMAP_SYSCALL_NUMBER, nullptr, tls_size_with_addr, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + // We cannot check the return value with MAP_FAILED as that is the return + // of the mmap function and not the mmap syscall. + if (mmap_retval < 0 && static_cast(mmap_retval) > -app.page_size) + syscall_impl(SYS_exit, 1); + uintptr_t *tls_addr = reinterpret_cast(mmap_retval); + + // x86_64 TLS faces down from the thread pointer with the first entry + // pointing to the address of the first real TLS byte. + uintptr_t end_ptr = reinterpret_cast(tls_addr) + tls_size; + *reinterpret_cast(end_ptr) = end_ptr; + + inline_memcpy(reinterpret_cast(tls_addr), + reinterpret_cast(app.tls.address), + app.tls.init_size); + uintptr_t *stack_guard_addr = reinterpret_cast(end_ptr + 40); + // Setting the stack guard to a random value. + // We cannot call the get_random function here as the function sets errno on + // failure. Since errno is implemented via a thread local variable, we cannot + // use errno before TLS is setup. + long stack_guard_retval = + syscall_impl(SYS_getrandom, reinterpret_cast(stack_guard_addr), + sizeof(uint64_t), 0); + if (stack_guard_retval < 0) + syscall_impl(SYS_exit, 1); + + tls_descriptor = {tls_size_with_addr, reinterpret_cast(tls_addr), + end_ptr}; + return; +} + +void cleanup_tls(uintptr_t addr, uintptr_t size) { + if (size == 0) + return; + syscall_impl(SYS_munmap, addr, size); +} + +// Sets the thread pointer to |val|. Returns true on success, false on failure. +bool set_thread_ptr(uintptr_t val) { + return syscall_impl(SYS_arch_prctl, ARCH_SET_FS, val) != -1; +} +} // namespace LIBC_NAMESPACE -- GitLab From a8cb4f7273ac4ea6f9cc3c03ed65a32542e947fe Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Thu, 4 Jan 2024 12:59:31 -0800 Subject: [PATCH 042/728] [RISCV][llvm-mca] Fix failing strided-load-x0.s test 58f1640635feff282935153d295dfb4ea1818401 was committed but had a broken test that I did not add the updated version to the commit. This patch fixes the test. --- .../llvm-mca/RISCV/SiFive7/strided-load-x0.s | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-x0.s b/llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-x0.s index eace2ad12d65..19864bed8722 100644 --- a/llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-x0.s +++ b/llvm/test/tools/llvm-mca/RISCV/SiFive7/strided-load-x0.s @@ -37,13 +37,13 @@ vle64.v v1, (a1) # CHECK: Iterations: 1 # CHECK-NEXT: Instructions: 26 -# CHECK-NEXT: Total Cycles: 3546 +# CHECK-NEXT: Total Cycles: 234 # CHECK-NEXT: Total uOps: 26 # CHECK: Dispatch Width: 2 -# CHECK-NEXT: uOps Per Cycle: 0.01 -# CHECK-NEXT: IPC: 0.01 -# CHECK-NEXT: Block RThroughput: 3541.0 +# CHECK-NEXT: uOps Per Cycle: 0.11 +# CHECK-NEXT: IPC: 0.11 +# CHECK-NEXT: Block RThroughput: 229.0 # CHECK: Instruction Info: # CHECK-NEXT: [1]: #uOps @@ -55,26 +55,26 @@ vle64.v v1, (a1) # CHECK: [1] [2] [3] [4] [5] [6] Instructions: # CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e32, m1, tu, mu -# CHECK-NEXT: 1 515 513.00 * vlse8.v v1, (a1), a2 -# CHECK-NEXT: 1 259 257.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse16.v v1, (a1), a2 # CHECK-NEXT: 1 19 17.00 * vlse32.v v1, (a1), a2 -# CHECK-NEXT: 1 67 65.00 * vlse64.v v1, (a1), a2 -# CHECK-NEXT: 1 515 513.00 * vlse8.v v1, (a1), zero -# CHECK-NEXT: 1 259 257.00 * vlse16.v v1, (a1), zero +# CHECK-NEXT: 1 19 17.00 * vlse64.v v1, (a1), a2 +# CHECK-NEXT: 1 19 17.00 * vlse8.v v1, (a1), zero +# CHECK-NEXT: 1 19 17.00 * vlse16.v v1, (a1), zero # CHECK-NEXT: 1 19 17.00 * vlse32.v v1, (a1), zero -# CHECK-NEXT: 1 67 65.00 * vlse64.v v1, (a1), zero +# CHECK-NEXT: 1 19 17.00 * vlse64.v v1, (a1), zero # CHECK-NEXT: 1 4 2.00 * vle8.v v1, (a1) # CHECK-NEXT: 1 4 2.00 * vle16.v v1, (a1) # CHECK-NEXT: 1 4 3.00 * vle32.v v1, (a1) # CHECK-NEXT: 1 4 5.00 * vle64.v v1, (a1) # CHECK-NEXT: 1 3 1.00 U vsetvli zero, zero, e64, m1, tu, mu -# CHECK-NEXT: 1 515 513.00 * vlse8.v v1, (a1), a2 -# CHECK-NEXT: 1 259 257.00 * vlse16.v v1, (a1), a2 -# CHECK-NEXT: 1 131 129.00 * vlse32.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse8.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse16.v v1, (a1), a2 +# CHECK-NEXT: 1 11 9.00 * vlse32.v v1, (a1), a2 # CHECK-NEXT: 1 11 9.00 * vlse64.v v1, (a1), a2 -# CHECK-NEXT: 1 515 513.00 * vlse8.v v1, (a1), zero -# CHECK-NEXT: 1 259 257.00 * vlse16.v v1, (a1), zero -# CHECK-NEXT: 1 131 129.00 * vlse32.v v1, (a1), zero +# CHECK-NEXT: 1 11 9.00 * vlse8.v v1, (a1), zero +# CHECK-NEXT: 1 11 9.00 * vlse16.v v1, (a1), zero +# CHECK-NEXT: 1 11 9.00 * vlse32.v v1, (a1), zero # CHECK-NEXT: 1 11 9.00 * vlse64.v v1, (a1), zero # CHECK-NEXT: 1 4 2.00 * vle8.v v1, (a1) # CHECK-NEXT: 1 4 2.00 * vle16.v v1, (a1) @@ -93,31 +93,31 @@ vle64.v v1, (a1) # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] -# CHECK-NEXT: - - 2.00 - - 24.00 3541.00 - +# CHECK-NEXT: - - 2.00 - - 24.00 229.00 - # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1] [2] [3] [4] [5] [6] [7] Instructions: # CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e32, m1, tu, mu -# CHECK-NEXT: - - - - - 1.00 513.00 - vlse8.v v1, (a1), a2 -# CHECK-NEXT: - - - - - 1.00 257.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse16.v v1, (a1), a2 # CHECK-NEXT: - - - - - 1.00 17.00 - vlse32.v v1, (a1), a2 -# CHECK-NEXT: - - - - - 1.00 65.00 - vlse64.v v1, (a1), a2 -# CHECK-NEXT: - - - - - 1.00 513.00 - vlse8.v v1, (a1), zero -# CHECK-NEXT: - - - - - 1.00 257.00 - vlse16.v v1, (a1), zero +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse64.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse8.v v1, (a1), zero +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse16.v v1, (a1), zero # CHECK-NEXT: - - - - - 1.00 17.00 - vlse32.v v1, (a1), zero -# CHECK-NEXT: - - - - - 1.00 65.00 - vlse64.v v1, (a1), zero +# CHECK-NEXT: - - - - - 1.00 17.00 - vlse64.v v1, (a1), zero # CHECK-NEXT: - - - - - 1.00 2.00 - vle8.v v1, (a1) # CHECK-NEXT: - - - - - 1.00 2.00 - vle16.v v1, (a1) # CHECK-NEXT: - - - - - 1.00 3.00 - vle32.v v1, (a1) # CHECK-NEXT: - - - - - 1.00 5.00 - vle64.v v1, (a1) # CHECK-NEXT: - - 1.00 - - - - - vsetvli zero, zero, e64, m1, tu, mu -# CHECK-NEXT: - - - - - 1.00 513.00 - vlse8.v v1, (a1), a2 -# CHECK-NEXT: - - - - - 1.00 257.00 - vlse16.v v1, (a1), a2 -# CHECK-NEXT: - - - - - 1.00 129.00 - vlse32.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse8.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse16.v v1, (a1), a2 +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse32.v v1, (a1), a2 # CHECK-NEXT: - - - - - 1.00 9.00 - vlse64.v v1, (a1), a2 -# CHECK-NEXT: - - - - - 1.00 513.00 - vlse8.v v1, (a1), zero -# CHECK-NEXT: - - - - - 1.00 257.00 - vlse16.v v1, (a1), zero -# CHECK-NEXT: - - - - - 1.00 129.00 - vlse32.v v1, (a1), zero +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse8.v v1, (a1), zero +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse16.v v1, (a1), zero +# CHECK-NEXT: - - - - - 1.00 9.00 - vlse32.v v1, (a1), zero # CHECK-NEXT: - - - - - 1.00 9.00 - vlse64.v v1, (a1), zero # CHECK-NEXT: - - - - - 1.00 2.00 - vle8.v v1, (a1) # CHECK-NEXT: - - - - - 1.00 2.00 - vle16.v v1, (a1) -- GitLab From 40b4ac278e87e2b98e52094b72728936df3e92d9 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 4 Jan 2024 13:13:22 -0800 Subject: [PATCH 043/728] [ORC] Refactor executor symbol lookup to use ExecutorSymbolDef (NFC) (#76989) This migrates the dylib manager lookup and related APIs to replace ExecutorAddress with ExecutorSymbolDef so that in the future we can model JITSymbolFlags for these symbols. The current change should be NFC as we are only setting the Exported symbol flag. --- compiler-rt/lib/orc/executor_symbol_def.h | 151 ++++++++++++++++++ compiler-rt/lib/orc/tests/unit/CMakeLists.txt | 1 + .../tests/unit/executor_symbol_def_test.cpp | 19 +++ .../unit/simple_packed_serialization_test.cpp | 20 +-- .../unit/simple_packed_serialization_utils.h | 34 ++++ .../Orc/EPCGenericDylibManager.h | 7 +- .../Orc/Shared/ExecutorSymbolDef.h | 58 +++++++ .../ExecutionEngine/Orc/Shared/OrcRTBridge.h | 3 +- .../Orc/Shared/TargetProcessControlTypes.h | 3 +- .../SimpleExecutorDylibManager.h | 5 +- .../Orc/EPCDebugObjectRegistrar.cpp | 3 +- .../Orc/EPCDynamicLibrarySearchGenerator.cpp | 4 +- .../Orc/EPCGenericDylibManager.cpp | 10 +- .../Orc/ExecutorProcessControl.cpp | 6 +- .../Orc/LookupAndRecordAddrs.cpp | 2 +- .../SimpleExecutorDylibManager.cpp | 9 +- llvm/tools/lli/ForwardingMemoryManager.h | 9 +- 17 files changed, 300 insertions(+), 44 deletions(-) create mode 100644 compiler-rt/lib/orc/executor_symbol_def.h create mode 100644 compiler-rt/lib/orc/tests/unit/executor_symbol_def_test.cpp create mode 100644 compiler-rt/lib/orc/tests/unit/simple_packed_serialization_utils.h diff --git a/compiler-rt/lib/orc/executor_symbol_def.h b/compiler-rt/lib/orc/executor_symbol_def.h new file mode 100644 index 000000000000..454cefe525cf --- /dev/null +++ b/compiler-rt/lib/orc/executor_symbol_def.h @@ -0,0 +1,151 @@ +//===--------- ExecutorSymbolDef.h - (Addr, Flags) pair ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Represents a defining location for a symbol in the executing program. +// +// This file was derived from +// llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_EXECUTOR_SYMBOL_DEF_H +#define ORC_RT_EXECUTOR_SYMBOL_DEF_H + +#include "bitmask_enum.h" +#include "executor_address.h" +#include "simple_packed_serialization.h" + +namespace __orc_rt { + +/// Flags for symbols in the JIT. +class JITSymbolFlags { +public: + using UnderlyingType = uint8_t; + using TargetFlagsType = uint8_t; + + /// These values must be kept in sync with \c JITSymbolFlags in the JIT. + enum FlagNames : UnderlyingType { + None = 0, + HasError = 1U << 0, + Weak = 1U << 1, + Common = 1U << 2, + Absolute = 1U << 3, + Exported = 1U << 4, + Callable = 1U << 5, + MaterializationSideEffectsOnly = 1U << 6, + ORC_RT_MARK_AS_BITMASK_ENUM( // LargestValue = + MaterializationSideEffectsOnly) + }; + + /// Default-construct a JITSymbolFlags instance. + JITSymbolFlags() = default; + + /// Construct a JITSymbolFlags instance from the given flags and target + /// flags. + JITSymbolFlags(FlagNames Flags, TargetFlagsType TargetFlags) + : TargetFlags(TargetFlags), Flags(Flags) {} + + bool operator==(const JITSymbolFlags &RHS) const { + return Flags == RHS.Flags && TargetFlags == RHS.TargetFlags; + } + + /// Get the underlying flags value as an integer. + UnderlyingType getRawFlagsValue() const { + return static_cast(Flags); + } + + /// Return a reference to the target-specific flags. + TargetFlagsType &getTargetFlags() { return TargetFlags; } + + /// Return a reference to the target-specific flags. + const TargetFlagsType &getTargetFlags() const { return TargetFlags; } + +private: + TargetFlagsType TargetFlags = 0; + FlagNames Flags = None; +}; + +/// Represents a defining location for a JIT symbol. +class ExecutorSymbolDef { +public: + ExecutorSymbolDef() = default; + ExecutorSymbolDef(ExecutorAddr Addr, JITSymbolFlags Flags) + : Addr(Addr), Flags(Flags) {} + + const ExecutorAddr &getAddress() const { return Addr; } + + const JITSymbolFlags &getFlags() const { return Flags; } + + friend bool operator==(const ExecutorSymbolDef &LHS, + const ExecutorSymbolDef &RHS) { + return LHS.getAddress() == RHS.getAddress() && + LHS.getFlags() == RHS.getFlags(); + } + +private: + ExecutorAddr Addr; + JITSymbolFlags Flags; +}; + +using SPSJITSymbolFlags = + SPSTuple; + +/// SPS serializatior for JITSymbolFlags. +template <> class SPSSerializationTraits { + using FlagsArgList = SPSJITSymbolFlags::AsArgList; + +public: + static size_t size(const JITSymbolFlags &F) { + return FlagsArgList::size(F.getRawFlagsValue(), F.getTargetFlags()); + } + + static bool serialize(SPSOutputBuffer &BOB, const JITSymbolFlags &F) { + return FlagsArgList::serialize(BOB, F.getRawFlagsValue(), + F.getTargetFlags()); + } + + static bool deserialize(SPSInputBuffer &BIB, JITSymbolFlags &F) { + JITSymbolFlags::UnderlyingType RawFlags; + JITSymbolFlags::TargetFlagsType TargetFlags; + if (!FlagsArgList::deserialize(BIB, RawFlags, TargetFlags)) + return false; + F = JITSymbolFlags{static_cast(RawFlags), + TargetFlags}; + return true; + } +}; + +using SPSExecutorSymbolDef = SPSTuple; + +/// SPS serializatior for ExecutorSymbolDef. +template <> +class SPSSerializationTraits { + using DefArgList = SPSExecutorSymbolDef::AsArgList; + +public: + static size_t size(const ExecutorSymbolDef &ESD) { + return DefArgList::size(ESD.getAddress(), ESD.getFlags()); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorSymbolDef &ESD) { + return DefArgList::serialize(BOB, ESD.getAddress(), ESD.getFlags()); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorSymbolDef &ESD) { + ExecutorAddr Addr; + JITSymbolFlags Flags; + if (!DefArgList::deserialize(BIB, Addr, Flags)) + return false; + ESD = ExecutorSymbolDef{Addr, Flags}; + return true; + } +}; + +} // End namespace __orc_rt + +#endif // ORC_RT_EXECUTOR_SYMBOL_DEF_H diff --git a/compiler-rt/lib/orc/tests/unit/CMakeLists.txt b/compiler-rt/lib/orc/tests/unit/CMakeLists.txt index 81df7e803bc5..aec689a407be 100644 --- a/compiler-rt/lib/orc/tests/unit/CMakeLists.txt +++ b/compiler-rt/lib/orc/tests/unit/CMakeLists.txt @@ -5,6 +5,7 @@ set(UNITTEST_SOURCES endian_test.cpp error_test.cpp executor_address_test.cpp + executor_symbol_def_test.cpp extensible_rtti_test.cpp interval_map_test.cpp interval_set_test.cpp diff --git a/compiler-rt/lib/orc/tests/unit/executor_symbol_def_test.cpp b/compiler-rt/lib/orc/tests/unit/executor_symbol_def_test.cpp new file mode 100644 index 000000000000..181091ca1e60 --- /dev/null +++ b/compiler-rt/lib/orc/tests/unit/executor_symbol_def_test.cpp @@ -0,0 +1,19 @@ +//===-- executor_symbol_def_test.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "executor_symbol_def.h" +#include "simple_packed_serialization_utils.h" +#include "gtest/gtest.h" + +using namespace __orc_rt; + +TEST(ExecutorSymbolDefTest, Serialization) { + blobSerializationRoundTrip(ExecutorSymbolDef{}); + blobSerializationRoundTrip( + ExecutorSymbolDef{ExecutorAddr{0x70}, {JITSymbolFlags::Callable, 9}}); +} \ No newline at end of file diff --git a/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_test.cpp b/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_test.cpp index e7a78062b210..397114b4017e 100644 --- a/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_test.cpp +++ b/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_test.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "simple_packed_serialization.h" +#include "simple_packed_serialization_utils.h" #include "gtest/gtest.h" using namespace __orc_rt; @@ -48,25 +49,6 @@ TEST(SimplePackedSerializationTest, SPSInputBuffer) { EXPECT_FALSE(IB.read(&C, 1)); } -template -static void blobSerializationRoundTrip(const T &Value) { - using BST = SPSSerializationTraits; - - size_t Size = BST::size(Value); - auto Buffer = std::make_unique(Size); - SPSOutputBuffer OB(Buffer.get(), Size); - - EXPECT_TRUE(BST::serialize(OB, Value)); - - SPSInputBuffer IB(Buffer.get(), Size); - - T DSValue; - EXPECT_TRUE(BST::deserialize(IB, DSValue)); - - EXPECT_EQ(Value, DSValue) - << "Incorrect value after serialization/deserialization round-trip"; -} - template static void testFixedIntegralTypeSerialization() { blobSerializationRoundTrip(0); blobSerializationRoundTrip(static_cast(1)); diff --git a/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_utils.h b/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_utils.h new file mode 100644 index 000000000000..746be43d250b --- /dev/null +++ b/compiler-rt/lib/orc/tests/unit/simple_packed_serialization_utils.h @@ -0,0 +1,34 @@ +//===-- simple_packed_serialization_utils.h -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_TEST_SIMPLE_PACKED_SERIALIZATION_UTILS_H +#define ORC_RT_TEST_SIMPLE_PACKED_SERIALIZATION_UTILS_H + +#include "simple_packed_serialization.h" +#include "gtest/gtest.h" + +template +static void blobSerializationRoundTrip(const T &Value) { + using BST = __orc_rt::SPSSerializationTraits; + + size_t Size = BST::size(Value); + auto Buffer = std::make_unique(Size); + __orc_rt::SPSOutputBuffer OB(Buffer.get(), Size); + + EXPECT_TRUE(BST::serialize(OB, Value)); + + __orc_rt::SPSInputBuffer IB(Buffer.get(), Size); + + T DSValue; + EXPECT_TRUE(BST::deserialize(IB, DSValue)); + + EXPECT_EQ(Value, DSValue) + << "Incorrect value after serialization/deserialization round-trip"; +} + +#endif // ORC_RT_TEST_SIMPLE_PACKED_SERIALIZATION_UTILS_H \ No newline at end of file diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h index 02e580c86f54..6ee2deef04d0 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h @@ -19,6 +19,7 @@ #define LLVM_EXECUTIONENGINE_ORC_EPCGENERICDYLIBMANAGER_H #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" namespace llvm { @@ -49,11 +50,11 @@ public: Expected open(StringRef Path, uint64_t Mode); /// Looks up symbols within the given dylib. - Expected> lookup(tpctypes::DylibHandle H, - const SymbolLookupSet &Lookup); + Expected> + lookup(tpctypes::DylibHandle H, const SymbolLookupSet &Lookup); /// Looks up symbols within the given dylib. - Expected> + Expected> lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &Lookup); private: diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h index 5c58a7255ebd..68ccdf83bd12 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h @@ -15,6 +15,7 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" namespace llvm { namespace orc { @@ -48,6 +49,63 @@ private: JITSymbolFlags Flags; }; +namespace shared { + +using SPSJITSymbolFlags = + SPSTuple; + +/// SPS serializatior for JITSymbolFlags. +template <> class SPSSerializationTraits { + using FlagsArgList = SPSJITSymbolFlags::AsArgList; + +public: + static size_t size(const JITSymbolFlags &F) { + return FlagsArgList::size(F.getRawFlagsValue(), F.getTargetFlags()); + } + + static bool serialize(SPSOutputBuffer &BOB, const JITSymbolFlags &F) { + return FlagsArgList::serialize(BOB, F.getRawFlagsValue(), + F.getTargetFlags()); + } + + static bool deserialize(SPSInputBuffer &BIB, JITSymbolFlags &F) { + JITSymbolFlags::UnderlyingType RawFlags; + JITSymbolFlags::TargetFlagsType TargetFlags; + if (!FlagsArgList::deserialize(BIB, RawFlags, TargetFlags)) + return false; + F = JITSymbolFlags{static_cast(RawFlags), + TargetFlags}; + return true; + } +}; + +using SPSExecutorSymbolDef = SPSTuple; + +/// SPS serializatior for ExecutorSymbolDef. +template <> +class SPSSerializationTraits { + using DefArgList = SPSExecutorSymbolDef::AsArgList; + +public: + static size_t size(const ExecutorSymbolDef &ESD) { + return DefArgList::size(ESD.getAddress(), ESD.getFlags()); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorSymbolDef &ESD) { + return DefArgList::serialize(BOB, ESD.getAddress(), ESD.getFlags()); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorSymbolDef &ESD) { + ExecutorAddr Addr; + JITSymbolFlags Flags; + if (!DefArgList::deserialize(BIB, Addr, Flags)) + return false; + ESD = ExecutorSymbolDef{Addr, Flags}; + return true; + } +}; + +} // End namespace shared. } // End namespace orc. } // End namespace llvm. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h index 8e336ca03eaf..0c549bcbf013 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_SHARED_ORCRTBRIDGE_H #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" @@ -54,7 +55,7 @@ using SPSSimpleExecutorDylibManagerOpenSignature = shared::SPSString, uint64_t); using SPSSimpleExecutorDylibManagerLookupSignature = - shared::SPSExpected>( + shared::SPSExpected>( shared::SPSExecutorAddr, shared::SPSExecutorAddr, shared::SPSRemoteSymbolLookupSet); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h index 7322674559c9..e91d8d926d88 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h @@ -19,6 +19,7 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h" #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" @@ -113,7 +114,7 @@ struct PointerWrite { /// A handle used to represent a loaded dylib in the target process. using DylibHandle = ExecutorAddr; -using LookupResult = std::vector; +using LookupResult = std::vector; } // end namespace tpctypes diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h index fd4504cfb7fb..00fd84e3ec14 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h @@ -18,6 +18,7 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" #include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" @@ -37,8 +38,8 @@ public: virtual ~SimpleExecutorDylibManager(); Expected open(const std::string &Path, uint64_t Mode); - Expected> lookup(tpctypes::DylibHandle H, - const RemoteSymbolLookupSet &L); + Expected> + lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &L); Error shutdown() override; void addBootstrapSymbols(StringMap &M) override; diff --git a/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp b/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp index b8969de54936..acd7e5a409fc 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp @@ -45,7 +45,8 @@ Expected> createJITLoaderGDBRegistrar( assert((*Result)[0].size() == 1 && "Unexpected number of addresses in result"); - return std::make_unique(ES, (*Result)[0][0]); + ExecutorAddr RegisterAddr = (*Result)[0][0].getAddress(); + return std::make_unique(ES, RegisterAddr); } Error EPCDebugObjectRegistrar::registerDebugObject(ExecutorAddrRange TargetMem, diff --git a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp index 46e16a55c7e1..2e2e7a9c5f32 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp @@ -52,8 +52,8 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate( auto ResultI = Result->front().begin(); for (auto &KV : LookupSymbols) { - if (*ResultI) - NewSymbols[KV.first] = {*ResultI, JITSymbolFlags::Exported}; + if (ResultI->getAddress()) + NewSymbols[KV.first] = *ResultI; ++ResultI; } diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp index e70749cdfab2..da185c80c6c7 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp @@ -81,10 +81,11 @@ Expected EPCGenericDylibManager::open(StringRef Path, return H; } -Expected> +Expected> EPCGenericDylibManager::lookup(tpctypes::DylibHandle H, const SymbolLookupSet &Lookup) { - Expected> Result((std::vector())); + Expected> Result( + (std::vector())); if (auto Err = EPC.callSPSWrapper( SAs.Lookup, Result, SAs.Instance, H, Lookup)) @@ -92,10 +93,11 @@ EPCGenericDylibManager::lookup(tpctypes::DylibHandle H, return Result; } -Expected> +Expected> EPCGenericDylibManager::lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &Lookup) { - Expected> Result((std::vector())); + Expected> Result( + (std::vector())); if (auto Err = EPC.callSPSWrapper( SAs.Lookup, Result, SAs.Instance, H, Lookup)) diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index ad27deff38d9..f0c551cd7780 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -95,7 +95,7 @@ SelfExecutorProcessControl::lookupSymbols(ArrayRef Request) { for (auto &Elem : Request) { sys::DynamicLibrary Dylib(Elem.Handle.toPtr()); - R.push_back(std::vector()); + R.push_back(std::vector()); for (auto &KV : Elem.Symbols) { auto &Sym = KV.first; std::string Tmp((*Sym).data() + !!GlobalManglingPrefix, @@ -107,7 +107,9 @@ SelfExecutorProcessControl::lookupSymbols(ArrayRef Request) { MissingSymbols.push_back(Sym); return make_error(SSP, std::move(MissingSymbols)); } - R.back().push_back(ExecutorAddr::fromPtr(Addr)); + // FIXME: determine accurate JITSymbolFlags. + R.back().push_back( + {ExecutorAddr::fromPtr(Addr), JITSymbolFlags::Exported}); } } diff --git a/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp b/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp index 75075c5c2a22..a369e1b53382 100644 --- a/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp @@ -73,7 +73,7 @@ Error lookupAndRecordAddrs( inconvertibleErrorCode()); for (unsigned I = 0; I != Pairs.size(); ++I) - *Pairs[I].second = Result->front()[I]; + *Pairs[I].second = Result->front()[I].getAddress(); return Error::success(); } diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp index cb11b68e2719..b7e256a826ca 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp @@ -40,10 +40,10 @@ SimpleExecutorDylibManager::open(const std::string &Path, uint64_t Mode) { return H; } -Expected> +Expected> SimpleExecutorDylibManager::lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &L) { - std::vector Result; + std::vector Result; auto DL = sys::DynamicLibrary(H.toPtr()); for (const auto &E : L) { @@ -52,7 +52,7 @@ SimpleExecutorDylibManager::lookup(tpctypes::DylibHandle H, return make_error("Required address for empty symbol \"\"", inconvertibleErrorCode()); else - Result.push_back(ExecutorAddr()); + Result.push_back(ExecutorSymbolDef()); } else { const char *DemangledSymName = E.Name.c_str(); @@ -70,7 +70,8 @@ SimpleExecutorDylibManager::lookup(tpctypes::DylibHandle H, DemangledSymName, inconvertibleErrorCode()); - Result.push_back(ExecutorAddr::fromPtr(Addr)); + // FIXME: determine accurate JITSymbolFlags. + Result.push_back({ExecutorAddr::fromPtr(Addr), JITSymbolFlags::Exported}); } } diff --git a/llvm/tools/lli/ForwardingMemoryManager.h b/llvm/tools/lli/ForwardingMemoryManager.h index f1de7a153a27..2cc669953ee6 100644 --- a/llvm/tools/lli/ForwardingMemoryManager.h +++ b/llvm/tools/lli/ForwardingMemoryManager.h @@ -105,13 +105,14 @@ public: JITSymbol findSymbol(const std::string &Name) override { orc::RemoteSymbolLookupSet R; R.push_back({std::move(Name), false}); - if (auto Addrs = DylibMgr.lookup(H, R)) { - if (Addrs->size() != 1) + if (auto Syms = DylibMgr.lookup(H, R)) { + if (Syms->size() != 1) return make_error("Unexpected remote lookup result", inconvertibleErrorCode()); - return JITSymbol(Addrs->front().getValue(), JITSymbolFlags::Exported); + return JITSymbol(Syms->front().getAddress().getValue(), + Syms->front().getFlags()); } else - return Addrs.takeError(); + return Syms.takeError(); } JITSymbol findSymbolInLogicalDylib(const std::string &Name) override { -- GitLab From 6af713ae170c34f0561f19e594266ce2a2af343b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Thu, 4 Jan 2024 23:18:53 +0200 Subject: [PATCH 044/728] [LLD] [MinGW] Remove an unnecessary include of unistd.h. NFC. (#76953) This was present since when the MinGW LLD frontend first was merged in 894dbbe8eb0e3d8fc7d8746a76d2380f0c2b6c0f, where it most probably was a leftover from earlier evolution of the patch. --- lld/MinGW/Driver.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp index 94f0ae7993e6..5ba1bf0e4b4e 100644 --- a/lld/MinGW/Driver.cpp +++ b/lld/MinGW/Driver.cpp @@ -46,10 +46,6 @@ #include "llvm/TargetParser/Triple.h" #include -#if !defined(_MSC_VER) && !defined(__MINGW32__) -#include -#endif - using namespace lld; using namespace llvm::opt; using namespace llvm; -- GitLab From dd047c5b64944bae830b9fecf53f8d11ff41386e Mon Sep 17 00:00:00 2001 From: madanial0 <118996571+madanial0@users.noreply.github.com> Date: Thu, 4 Jan 2024 16:23:01 -0500 Subject: [PATCH 045/728] [Flang] remove setting lowerbound for non existing dimension in shift runtime test case (NFC) (#76990) The shift2 array only has 1 dimension but the lower bound for a second dimension is being set causing a seg fault on AIX. Co-authored-by: Mark Danial --- flang/unittests/Runtime/Transformational.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/flang/unittests/Runtime/Transformational.cpp b/flang/unittests/Runtime/Transformational.cpp index 942a20647ded..5678ea251577 100644 --- a/flang/unittests/Runtime/Transformational.cpp +++ b/flang/unittests/Runtime/Transformational.cpp @@ -250,7 +250,6 @@ TEST(Transformational, Shifts) { auto shift2{MakeArray( std::vector{2}, std::vector{1, -1})}; shift2->GetDimension(0).SetLowerBound(-1); // shouldn't matter - shift2->GetDimension(1).SetLowerBound(2); RTNAME(Cshift)(result, *array, *shift2, 2, __FILE__, __LINE__); EXPECT_EQ(result.type(), array->type()); EXPECT_EQ(result.rank(), 2); -- GitLab From 7a05c0931f1a0ade9abc447c872a3cc7c8a37dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Thu, 4 Jan 2024 16:03:21 +0200 Subject: [PATCH 046/728] [LLD] [COFF] Fix option name references in Config.h. NFC. These options have been named /lldltocache: and /lldltocachepolicy: since they were added in 052e855e2bea78dcfbb2807acee829b56d56a729; the comment was wrong from the original commit. --- lld/COFF/Config.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h index 48c48cefe91d..018f03b211e4 100644 --- a/lld/COFF/Config.h +++ b/lld/COFF/Config.h @@ -186,9 +186,9 @@ struct Configuration { // Used for /opt:lldltopartitions=N unsigned ltoPartitions = 1; - // Used for /opt:lldltocache=path + // Used for /lldltocache=path StringRef ltoCache; - // Used for /opt:lldltocachepolicy=policy + // Used for /lldltocachepolicy=policy llvm::CachePruningPolicy ltoCachePolicy; // Used for /opt:[no]ltodebugpassmanager -- GitLab From cb7fe9ad4c3103d90c20d55819a9e69ab66ab3d0 Mon Sep 17 00:00:00 2001 From: Valery Pykhtin Date: Thu, 4 Jan 2024 22:58:32 +0100 Subject: [PATCH 047/728] [ASAN][AMDGPU] Make address sanitizer checks more efficient for the divergent target. (#72247) Address sanitizer checks for AMDGPU target in non-recovery mode aren't quite efficient at the moment which can be illustrated with a program: ``` instr_before; load ptr1; instr_in_the_middle; load ptr2; instr_after; ``` ASAN generates the following instrumentation: ``` instr_before; if (sanity_check_passed(ptr1)) load ptr1; instr_in_the_middle; if (sanity_check_passed(ptr2)) load ptr2; instr_after; else // ASAN report block 2 __asan_report(ptr2); // wave terminates unreachable; else // ASAN report block 1 __asan_report(ptr1); // wave terminates unreachable; ``` Each sanitizer check is treated as a non-uniform condition (and this is true because some lanes may pass the check and some don't). This results in the program above: basically normal program flow is continued in _then_ blocks. This way it allows lanes that pass all sanity checks to complete the program and then the wave terminates at the first reporting _else_ block. For each _else_ block it has to keep execmask and pointer value to report error consuming tons (megatons!) of registers which are live till the program end. This patch changes the behavior on a failing sanity check: instead of waiting when passing lanes reach program end report error and terminate as soon as any lane has violated the sanity check. Sanity check condition is treated uniform with this approach and the resulting program looks much like ordinary CPU code: ``` instr_before; if (any_lane_violated(sanity_check_passed(ptr1))) // ASAN report block 1 __asan_report(ptr1); // abort the program unreachable; load ptr1; instr_in_the_middle; if (any_lane_violated(sanity_check_passed(ptr2))) // ASAN report block 2 __asan_report(ptr2); // abort the program unreachable; load ptr2; instr_after; ``` However it has to use a trick to pass structurizer and some later passes: ASAN check is generated like in recovery mode but reporting function aborts, that is standard _unreachable_ instruction isn't used: ``` ... if (any_lane_violated(sanity_check_passed(ptr1))) // ASAN report block 1 __asan_report(ptr1); // abort the program // pretend we're going to continue the program load ptr1; ... ``` This may create some undesirable effects: 1. Register allocator generates a lot of code for save/restore registers for asan_report call. This may potentially bloat the code since we have a report block for every accessed pointer. 2. Loop invariant code in report blocks is hoisted into a loop preheader. I'm not sure but probably this can be solved using block frequency information, but most likely this isn't a problem at all. These problems are to be addressed later. ### Flattening address sanitizer check In order to simplify divergent CFG this patch also changes the instrumentation code from: ``` uint64_t address = ptr; sbyte *shadow_address = MemToShadow(address); sbyte shadow_value = *shadow_address; if (shadow_value) { sbyte last_accessed_byte = (address & 7) + kAccessSize - 1; if (last_accessed_byte >= shadow_value) { ReportError(address, kAccessSize, kIsWrite); abort(); } } ``` to ``` uint64_t address = ptr; sbyte *shadow_address = MemToShadow(address); sbyte shadow_value = *shadow_address; sbyte last_accessed_byte = (address & 7) + kAccessSize - 1; if (shadow_value && last_accessed_byte >= shadow_value) { ReportError(address, kAccessSize, kIsWrite); abort(); } ``` It saves one _if_ which really avoids very few instructions and their latency can be hidden by the load from shadow memory. --- .../Instrumentation/AddressSanitizer.cpp | 38 +++- .../asan_instrument_constant_address_space.ll | 74 ++++---- .../asan_instrument_generic_address_space.ll | 172 ++++++++++-------- .../asan_instrument_global_address_space.ll | 148 ++++++++------- 4 files changed, 259 insertions(+), 173 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index afb0e6cd1548..d4f5bf8c3935 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -174,6 +174,8 @@ const char kAsanAllocasUnpoison[] = "__asan_allocas_unpoison"; const char kAMDGPUAddressSharedName[] = "llvm.amdgcn.is.shared"; const char kAMDGPUAddressPrivateName[] = "llvm.amdgcn.is.private"; +const char kAMDGPUBallotName[] = "llvm.amdgcn.ballot.i64"; +const char kAMDGPUUnreachableName[] = "llvm.amdgcn.unreachable"; // Accesses sizes are powers of two: 1, 2, 4, 8, 16. static const size_t kNumberOfAccessSizes = 5; @@ -699,6 +701,8 @@ struct AddressSanitizer { Instruction *InsertBefore, Value *Addr, uint32_t TypeStoreSize, bool IsWrite, Value *SizeArgument); + Instruction *genAMDGPUReportBlock(IRBuilder<> &IRB, Value *Cond, + bool Recover); void instrumentUnusualSizeOrAlignment(Instruction *I, Instruction *InsertBefore, Value *Addr, TypeSize TypeStoreSize, bool IsWrite, @@ -1721,6 +1725,30 @@ Instruction *AddressSanitizer::instrumentAMDGPUAddress( return InsertBefore; } +Instruction *AddressSanitizer::genAMDGPUReportBlock(IRBuilder<> &IRB, + Value *Cond, bool Recover) { + Module &M = *IRB.GetInsertBlock()->getModule(); + Value *ReportCond = Cond; + if (!Recover) { + auto Ballot = M.getOrInsertFunction(kAMDGPUBallotName, IRB.getInt64Ty(), + IRB.getInt1Ty()); + ReportCond = IRB.CreateIsNotNull(IRB.CreateCall(Ballot, {Cond})); + } + + auto *Trm = + SplitBlockAndInsertIfThen(ReportCond, &*IRB.GetInsertPoint(), false, + MDBuilder(*C).createBranchWeights(1, 100000)); + Trm->getParent()->setName("asan.report"); + + if (Recover) + return Trm; + + Trm = SplitBlockAndInsertIfThen(Cond, Trm, false); + IRB.SetInsertPoint(Trm); + return IRB.CreateCall( + M.getOrInsertFunction(kAMDGPUUnreachableName, IRB.getVoidTy()), {}); +} + void AddressSanitizer::instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, MaybeAlign Alignment, @@ -1772,7 +1800,15 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns, size_t Granularity = 1ULL << Mapping.Scale; Instruction *CrashTerm = nullptr; - if (ClAlwaysSlowPath || (TypeStoreSize < 8 * Granularity)) { + bool GenSlowPath = (ClAlwaysSlowPath || (TypeStoreSize < 8 * Granularity)); + + if (TargetTriple.isAMDGCN()) { + if (GenSlowPath) { + auto *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeStoreSize); + Cmp = IRB.CreateAnd(Cmp, Cmp2); + } + CrashTerm = genAMDGPUReportBlock(IRB, Cmp, Recover); + } else if (GenSlowPath) { // We use branch weights for the slow path check, to indicate that the slow // path is rarely taken. This seems to be the case for SPEC benchmarks. Instruction *CheckTerm = SplitBlockAndInsertIfThen( diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll index 4ce337e2b68e..cdead5b26c7e 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll @@ -18,17 +18,23 @@ define protected amdgpu_kernel void @constant_load(i64 %i) sanitize_address { ; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; CHECK-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF2:![0-9]+]] -; CHECK: 6: -; CHECK-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 7 -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 3 -; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8 -; CHECK-NEXT: [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]] -; CHECK-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12]] -; CHECK: 11: -; CHECK-NEXT: call void @__asan_report_load4(i64 [[TMP0]]) #[[ATTR3:[0-9]+]] -; CHECK-NEXT: unreachable -; CHECK: 12: +; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[TMP0]], 7 +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 3 +; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp sge i8 [[TMP8]], [[TMP4]] +; CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP5]], [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i64 [[TMP11]], 0 +; CHECK-NEXT: br i1 [[TMP12]], label [[ASAN_REPORT:%.*]], label [[TMP15:%.*]], !prof [[PROF2:![0-9]+]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP10]], label [[TMP13:%.*]], label [[TMP14:%.*]] +; CHECK: 13: +; CHECK-NEXT: call void @__asan_report_load4(i64 [[TMP0]]) #[[ATTR5:[0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP14]] +; CHECK: 14: +; CHECK-NEXT: br label [[TMP15]] +; CHECK: 15: ; CHECK-NEXT: [[Q:%.*]] = load i32, ptr addrspace(4) [[A]], align 4 ; CHECK-NEXT: ret void ; @@ -42,19 +48,16 @@ define protected amdgpu_kernel void @constant_load(i64 %i) sanitize_address { ; RECOV-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; RECOV-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; RECOV-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; RECOV-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF2:![0-9]+]] -; RECOV: 6: -; RECOV-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 7 -; RECOV-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 3 -; RECOV-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8 -; RECOV-NEXT: [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]] -; RECOV-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12:%.*]] -; RECOV: 11: +; RECOV-NEXT: [[TMP6:%.*]] = and i64 [[TMP0]], 7 +; RECOV-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 3 +; RECOV-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8 +; RECOV-NEXT: [[TMP9:%.*]] = icmp sge i8 [[TMP8]], [[TMP4]] +; RECOV-NEXT: [[TMP10:%.*]] = and i1 [[TMP5]], [[TMP9]] +; RECOV-NEXT: br i1 [[TMP10]], label [[ASAN_REPORT:%.*]], label [[TMP11:%.*]], !prof [[PROF2:![0-9]+]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_load4_noabort(i64 [[TMP0]]) #[[ATTR3:[0-9]+]] -; RECOV-NEXT: br label [[TMP12]] -; RECOV: 12: -; RECOV-NEXT: br label [[TMP13]] -; RECOV: 13: +; RECOV-NEXT: br label [[TMP11]] +; RECOV: 11: ; RECOV-NEXT: [[Q:%.*]] = load i32, ptr addrspace(4) [[A]], align 4 ; RECOV-NEXT: ret void ; @@ -75,11 +78,18 @@ define protected amdgpu_kernel void @constant_load_8(i64 %i) sanitize_address { ; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; CHECK-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] -; CHECK: 6: -; CHECK-NEXT: call void @__asan_report_load8(i64 [[TMP0]]) #[[ATTR3]] -; CHECK-NEXT: unreachable -; CHECK: 7: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0 +; CHECK-NEXT: br i1 [[TMP7]], label [[ASAN_REPORT:%.*]], label [[TMP10:%.*]], !prof [[PROF2]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP5]], label [[TMP8:%.*]], label [[TMP9:%.*]] +; CHECK: 8: +; CHECK-NEXT: call void @__asan_report_load8(i64 [[TMP0]]) #[[ATTR5]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP9]] +; CHECK: 9: +; CHECK-NEXT: br label [[TMP10]] +; CHECK: 10: ; CHECK-NEXT: [[Q:%.*]] = load i64, ptr addrspace(4) [[A]], align 8 ; CHECK-NEXT: ret void ; @@ -93,11 +103,11 @@ define protected amdgpu_kernel void @constant_load_8(i64 %i) sanitize_address { ; RECOV-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; RECOV-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; RECOV-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; RECOV-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] -; RECOV: 6: +; RECOV-NEXT: br i1 [[TMP5]], label [[ASAN_REPORT:%.*]], label [[TMP6:%.*]], !prof [[PROF2]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_load8_noabort(i64 [[TMP0]]) #[[ATTR3]] -; RECOV-NEXT: br label [[TMP7]] -; RECOV: 7: +; RECOV-NEXT: br label [[TMP6]] +; RECOV: 6: ; RECOV-NEXT: [[Q:%.*]] = load i64, ptr addrspace(4) [[A]], align 8 ; RECOV-NEXT: ret void ; diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll index f86a5722c000..cb37ba24f1c7 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll @@ -13,7 +13,7 @@ define protected amdgpu_kernel void @generic_store(ptr addrspace(1) %p, i32 %i) ; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP18:%.*]] +; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP21:%.*]] ; CHECK: 4: ; CHECK-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -21,19 +21,25 @@ define protected amdgpu_kernel void @generic_store(ptr addrspace(1) %p, i32 %i) ; CHECK-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; CHECK-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP17:%.*]], !prof [[PROF0:![0-9]+]] -; CHECK: 11: -; CHECK-NEXT: [[TMP12:%.*]] = and i64 [[TMP5]], 7 -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 3 -; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8 -; CHECK-NEXT: [[TMP15:%.*]] = icmp sge i8 [[TMP14]], [[TMP9]] -; CHECK-NEXT: br i1 [[TMP15]], label [[TMP16:%.*]], label [[TMP17]] -; CHECK: 16: -; CHECK-NEXT: call void @__asan_report_store4(i64 [[TMP5]]) #[[ATTR3:[0-9]+]] -; CHECK-NEXT: unreachable -; CHECK: 17: -; CHECK-NEXT: br label [[TMP18]] +; CHECK-NEXT: [[TMP11:%.*]] = and i64 [[TMP5]], 7 +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], 3 +; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8 +; CHECK-NEXT: [[TMP14:%.*]] = icmp sge i8 [[TMP13]], [[TMP9]] +; CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP10]], [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP15]]) +; CHECK-NEXT: [[TMP17:%.*]] = icmp ne i64 [[TMP16]], 0 +; CHECK-NEXT: br i1 [[TMP17]], label [[ASAN_REPORT:%.*]], label [[TMP20:%.*]], !prof [[PROF0:![0-9]+]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP15]], label [[TMP18:%.*]], label [[TMP19:%.*]] ; CHECK: 18: +; CHECK-NEXT: call void @__asan_report_store4(i64 [[TMP5]]) #[[ATTR5:[0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP19]] +; CHECK: 19: +; CHECK-NEXT: br label [[TMP20]] +; CHECK: 20: +; CHECK-NEXT: br label [[TMP21]] +; CHECK: 21: ; CHECK-NEXT: store i32 0, ptr [[Q]], align 4 ; CHECK-NEXT: ret void ; @@ -45,7 +51,7 @@ define protected amdgpu_kernel void @generic_store(ptr addrspace(1) %p, i32 %i) ; RECOV-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; RECOV-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; RECOV-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP19:%.*]] +; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP17:%.*]] ; RECOV: 4: ; RECOV-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; RECOV-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -53,21 +59,18 @@ define protected amdgpu_kernel void @generic_store(ptr addrspace(1) %p, i32 %i) ; RECOV-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; RECOV-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; RECOV-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; RECOV-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP18:%.*]], !prof [[PROF0:![0-9]+]] -; RECOV: 11: -; RECOV-NEXT: [[TMP12:%.*]] = and i64 [[TMP5]], 7 -; RECOV-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 3 -; RECOV-NEXT: [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8 -; RECOV-NEXT: [[TMP15:%.*]] = icmp sge i8 [[TMP14]], [[TMP9]] -; RECOV-NEXT: br i1 [[TMP15]], label [[TMP16:%.*]], label [[TMP17:%.*]] -; RECOV: 16: +; RECOV-NEXT: [[TMP11:%.*]] = and i64 [[TMP5]], 7 +; RECOV-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], 3 +; RECOV-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8 +; RECOV-NEXT: [[TMP14:%.*]] = icmp sge i8 [[TMP13]], [[TMP9]] +; RECOV-NEXT: [[TMP15:%.*]] = and i1 [[TMP10]], [[TMP14]] +; RECOV-NEXT: br i1 [[TMP15]], label [[ASAN_REPORT:%.*]], label [[TMP16:%.*]], !prof [[PROF0:![0-9]+]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_store4_noabort(i64 [[TMP5]]) #[[ATTR3:[0-9]+]] +; RECOV-NEXT: br label [[TMP16]] +; RECOV: 16: ; RECOV-NEXT: br label [[TMP17]] ; RECOV: 17: -; RECOV-NEXT: br label [[TMP18]] -; RECOV: 18: -; RECOV-NEXT: br label [[TMP19]] -; RECOV: 19: ; RECOV-NEXT: store i32 0, ptr [[Q]], align 4 ; RECOV-NEXT: ret void ; @@ -87,7 +90,7 @@ define protected amdgpu_kernel void @generic_load(ptr addrspace(1) %p, i32 %i) s ; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP18:%.*]] +; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP21:%.*]] ; CHECK: 4: ; CHECK-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -95,19 +98,25 @@ define protected amdgpu_kernel void @generic_load(ptr addrspace(1) %p, i32 %i) s ; CHECK-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; CHECK-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP17:%.*]], !prof [[PROF0]] -; CHECK: 11: -; CHECK-NEXT: [[TMP12:%.*]] = and i64 [[TMP5]], 7 -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 3 -; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8 -; CHECK-NEXT: [[TMP15:%.*]] = icmp sge i8 [[TMP14]], [[TMP9]] -; CHECK-NEXT: br i1 [[TMP15]], label [[TMP16:%.*]], label [[TMP17]] -; CHECK: 16: -; CHECK-NEXT: call void @__asan_report_load4(i64 [[TMP5]]) #[[ATTR3]] -; CHECK-NEXT: unreachable -; CHECK: 17: -; CHECK-NEXT: br label [[TMP18]] +; CHECK-NEXT: [[TMP11:%.*]] = and i64 [[TMP5]], 7 +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], 3 +; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8 +; CHECK-NEXT: [[TMP14:%.*]] = icmp sge i8 [[TMP13]], [[TMP9]] +; CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP10]], [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP15]]) +; CHECK-NEXT: [[TMP17:%.*]] = icmp ne i64 [[TMP16]], 0 +; CHECK-NEXT: br i1 [[TMP17]], label [[ASAN_REPORT:%.*]], label [[TMP20:%.*]], !prof [[PROF0]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP15]], label [[TMP18:%.*]], label [[TMP19:%.*]] ; CHECK: 18: +; CHECK-NEXT: call void @__asan_report_load4(i64 [[TMP5]]) #[[ATTR5]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP19]] +; CHECK: 19: +; CHECK-NEXT: br label [[TMP20]] +; CHECK: 20: +; CHECK-NEXT: br label [[TMP21]] +; CHECK: 21: ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[Q]], align 4 ; CHECK-NEXT: ret void ; @@ -119,7 +128,7 @@ define protected amdgpu_kernel void @generic_load(ptr addrspace(1) %p, i32 %i) s ; RECOV-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; RECOV-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; RECOV-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP19:%.*]] +; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP17:%.*]] ; RECOV: 4: ; RECOV-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; RECOV-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -127,21 +136,18 @@ define protected amdgpu_kernel void @generic_load(ptr addrspace(1) %p, i32 %i) s ; RECOV-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; RECOV-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; RECOV-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; RECOV-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP18:%.*]], !prof [[PROF0]] -; RECOV: 11: -; RECOV-NEXT: [[TMP12:%.*]] = and i64 [[TMP5]], 7 -; RECOV-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], 3 -; RECOV-NEXT: [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8 -; RECOV-NEXT: [[TMP15:%.*]] = icmp sge i8 [[TMP14]], [[TMP9]] -; RECOV-NEXT: br i1 [[TMP15]], label [[TMP16:%.*]], label [[TMP17:%.*]] -; RECOV: 16: +; RECOV-NEXT: [[TMP11:%.*]] = and i64 [[TMP5]], 7 +; RECOV-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], 3 +; RECOV-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8 +; RECOV-NEXT: [[TMP14:%.*]] = icmp sge i8 [[TMP13]], [[TMP9]] +; RECOV-NEXT: [[TMP15:%.*]] = and i1 [[TMP10]], [[TMP14]] +; RECOV-NEXT: br i1 [[TMP15]], label [[ASAN_REPORT:%.*]], label [[TMP16:%.*]], !prof [[PROF0]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_load4_noabort(i64 [[TMP5]]) #[[ATTR3]] +; RECOV-NEXT: br label [[TMP16]] +; RECOV: 16: ; RECOV-NEXT: br label [[TMP17]] ; RECOV: 17: -; RECOV-NEXT: br label [[TMP18]] -; RECOV: 18: -; RECOV-NEXT: br label [[TMP19]] -; RECOV: 19: ; RECOV-NEXT: [[R:%.*]] = load i32, ptr [[Q]], align 4 ; RECOV-NEXT: ret void ; @@ -161,7 +167,7 @@ define protected amdgpu_kernel void @generic_store_8(ptr addrspace(1) %p) saniti ; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP13:%.*]] +; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP16:%.*]] ; CHECK: 4: ; CHECK-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -169,13 +175,20 @@ define protected amdgpu_kernel void @generic_store_8(ptr addrspace(1) %p) saniti ; CHECK-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; CHECK-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12:%.*]] -; CHECK: 11: -; CHECK-NEXT: call void @__asan_report_store8(i64 [[TMP5]]) #[[ATTR3]] -; CHECK-NEXT: unreachable -; CHECK: 12: -; CHECK-NEXT: br label [[TMP13]] +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i64 [[TMP11]], 0 +; CHECK-NEXT: br i1 [[TMP12]], label [[ASAN_REPORT:%.*]], label [[TMP15:%.*]], !prof [[PROF0]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP10]], label [[TMP13:%.*]], label [[TMP14:%.*]] ; CHECK: 13: +; CHECK-NEXT: call void @__asan_report_store8(i64 [[TMP5]]) #[[ATTR5]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP14]] +; CHECK: 14: +; CHECK-NEXT: br label [[TMP15]] +; CHECK: 15: +; CHECK-NEXT: br label [[TMP16]] +; CHECK: 16: ; CHECK-NEXT: store i64 0, ptr [[Q]], align 8 ; CHECK-NEXT: ret void ; @@ -187,7 +200,7 @@ define protected amdgpu_kernel void @generic_store_8(ptr addrspace(1) %p) saniti ; RECOV-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; RECOV-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; RECOV-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP13:%.*]] +; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP12:%.*]] ; RECOV: 4: ; RECOV-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; RECOV-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -195,13 +208,13 @@ define protected amdgpu_kernel void @generic_store_8(ptr addrspace(1) %p) saniti ; RECOV-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; RECOV-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; RECOV-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; RECOV-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12:%.*]] -; RECOV: 11: +; RECOV-NEXT: br i1 [[TMP10]], label [[ASAN_REPORT:%.*]], label [[TMP11:%.*]], !prof [[PROF0]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_store8_noabort(i64 [[TMP5]]) #[[ATTR3]] +; RECOV-NEXT: br label [[TMP11]] +; RECOV: 11: ; RECOV-NEXT: br label [[TMP12]] ; RECOV: 12: -; RECOV-NEXT: br label [[TMP13]] -; RECOV: 13: ; RECOV-NEXT: store i64 0, ptr [[Q]], align 8 ; RECOV-NEXT: ret void ; @@ -220,7 +233,7 @@ define protected amdgpu_kernel void @generic_load_8(ptr addrspace(1) %p) sanitiz ; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; CHECK-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP13:%.*]] +; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP16:%.*]] ; CHECK: 4: ; CHECK-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -228,13 +241,20 @@ define protected amdgpu_kernel void @generic_load_8(ptr addrspace(1) %p) sanitiz ; CHECK-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; CHECK-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; CHECK-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12:%.*]] -; CHECK: 11: -; CHECK-NEXT: call void @__asan_report_load8(i64 [[TMP5]]) #[[ATTR3]] -; CHECK-NEXT: unreachable -; CHECK: 12: -; CHECK-NEXT: br label [[TMP13]] +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i64 [[TMP11]], 0 +; CHECK-NEXT: br i1 [[TMP12]], label [[ASAN_REPORT:%.*]], label [[TMP15:%.*]], !prof [[PROF0]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP10]], label [[TMP13:%.*]], label [[TMP14:%.*]] ; CHECK: 13: +; CHECK-NEXT: call void @__asan_report_load8(i64 [[TMP5]]) #[[ATTR5]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP14]] +; CHECK: 14: +; CHECK-NEXT: br label [[TMP15]] +; CHECK: 15: +; CHECK-NEXT: br label [[TMP16]] +; CHECK: 16: ; CHECK-NEXT: [[R:%.*]] = load i64, ptr [[Q]], align 8 ; CHECK-NEXT: ret void ; @@ -246,7 +266,7 @@ define protected amdgpu_kernel void @generic_load_8(ptr addrspace(1) %p) sanitiz ; RECOV-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]]) ; RECOV-NEXT: [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]] ; RECOV-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true -; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP13:%.*]] +; RECOV-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP12:%.*]] ; RECOV: 4: ; RECOV-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64 ; RECOV-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 3 @@ -254,13 +274,13 @@ define protected amdgpu_kernel void @generic_load_8(ptr addrspace(1) %p) sanitiz ; RECOV-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr ; RECOV-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1 ; RECOV-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0 -; RECOV-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12:%.*]] -; RECOV: 11: +; RECOV-NEXT: br i1 [[TMP10]], label [[ASAN_REPORT:%.*]], label [[TMP11:%.*]], !prof [[PROF0]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_load8_noabort(i64 [[TMP5]]) #[[ATTR3]] +; RECOV-NEXT: br label [[TMP11]] +; RECOV: 11: ; RECOV-NEXT: br label [[TMP12]] ; RECOV: 12: -; RECOV-NEXT: br label [[TMP13]] -; RECOV: 13: ; RECOV-NEXT: [[R:%.*]] = load i64, ptr [[Q]], align 8 ; RECOV-NEXT: ret void ; diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll index 6b39ff6ff846..a954b173eb2a 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll @@ -14,17 +14,23 @@ define protected amdgpu_kernel void @global_store(ptr addrspace(1) %p, i32 %i) s ; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; CHECK-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF0:![0-9]+]] -; CHECK: 6: -; CHECK-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 7 -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 3 -; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8 -; CHECK-NEXT: [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]] -; CHECK-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12]] -; CHECK: 11: -; CHECK-NEXT: call void @__asan_report_store4(i64 [[TMP0]]) #[[ATTR3:[0-9]+]] -; CHECK-NEXT: unreachable -; CHECK: 12: +; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[TMP0]], 7 +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 3 +; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp sge i8 [[TMP8]], [[TMP4]] +; CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP5]], [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i64 [[TMP11]], 0 +; CHECK-NEXT: br i1 [[TMP12]], label [[ASAN_REPORT:%.*]], label [[TMP15:%.*]], !prof [[PROF0:![0-9]+]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP10]], label [[TMP13:%.*]], label [[TMP14:%.*]] +; CHECK: 13: +; CHECK-NEXT: call void @__asan_report_store4(i64 [[TMP0]]) #[[ATTR5:[0-9]+]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP14]] +; CHECK: 14: +; CHECK-NEXT: br label [[TMP15]] +; CHECK: 15: ; CHECK-NEXT: store i32 0, ptr addrspace(1) [[P]], align 4 ; CHECK-NEXT: ret void ; @@ -37,19 +43,16 @@ define protected amdgpu_kernel void @global_store(ptr addrspace(1) %p, i32 %i) s ; RECOV-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; RECOV-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; RECOV-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; RECOV-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF0:![0-9]+]] -; RECOV: 6: -; RECOV-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 7 -; RECOV-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 3 -; RECOV-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8 -; RECOV-NEXT: [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]] -; RECOV-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12:%.*]] -; RECOV: 11: +; RECOV-NEXT: [[TMP6:%.*]] = and i64 [[TMP0]], 7 +; RECOV-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 3 +; RECOV-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8 +; RECOV-NEXT: [[TMP9:%.*]] = icmp sge i8 [[TMP8]], [[TMP4]] +; RECOV-NEXT: [[TMP10:%.*]] = and i1 [[TMP5]], [[TMP9]] +; RECOV-NEXT: br i1 [[TMP10]], label [[ASAN_REPORT:%.*]], label [[TMP11:%.*]], !prof [[PROF0:![0-9]+]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_store4_noabort(i64 [[TMP0]]) #[[ATTR3:[0-9]+]] -; RECOV-NEXT: br label [[TMP12]] -; RECOV: 12: -; RECOV-NEXT: br label [[TMP13]] -; RECOV: 13: +; RECOV-NEXT: br label [[TMP11]] +; RECOV: 11: ; RECOV-NEXT: store i32 0, ptr addrspace(1) [[P]], align 4 ; RECOV-NEXT: ret void ; @@ -69,17 +72,23 @@ define protected amdgpu_kernel void @global_load(ptr addrspace(1) %p, i32 %i) sa ; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; CHECK-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF0]] -; CHECK: 6: -; CHECK-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 7 -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 3 -; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8 -; CHECK-NEXT: [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]] -; CHECK-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12]] -; CHECK: 11: -; CHECK-NEXT: call void @__asan_report_load4(i64 [[TMP0]]) #[[ATTR3]] -; CHECK-NEXT: unreachable -; CHECK: 12: +; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[TMP0]], 7 +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 3 +; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp sge i8 [[TMP8]], [[TMP4]] +; CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP5]], [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP10]]) +; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i64 [[TMP11]], 0 +; CHECK-NEXT: br i1 [[TMP12]], label [[ASAN_REPORT:%.*]], label [[TMP15:%.*]], !prof [[PROF0]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP10]], label [[TMP13:%.*]], label [[TMP14:%.*]] +; CHECK: 13: +; CHECK-NEXT: call void @__asan_report_load4(i64 [[TMP0]]) #[[ATTR5]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP14]] +; CHECK: 14: +; CHECK-NEXT: br label [[TMP15]] +; CHECK: 15: ; CHECK-NEXT: [[Q:%.*]] = load i32, ptr addrspace(1) [[P]], align 4 ; CHECK-NEXT: ret void ; @@ -92,19 +101,16 @@ define protected amdgpu_kernel void @global_load(ptr addrspace(1) %p, i32 %i) sa ; RECOV-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; RECOV-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; RECOV-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; RECOV-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP13:%.*]], !prof [[PROF0]] -; RECOV: 6: -; RECOV-NEXT: [[TMP7:%.*]] = and i64 [[TMP0]], 7 -; RECOV-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 3 -; RECOV-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8 -; RECOV-NEXT: [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]] -; RECOV-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12:%.*]] -; RECOV: 11: +; RECOV-NEXT: [[TMP6:%.*]] = and i64 [[TMP0]], 7 +; RECOV-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 3 +; RECOV-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8 +; RECOV-NEXT: [[TMP9:%.*]] = icmp sge i8 [[TMP8]], [[TMP4]] +; RECOV-NEXT: [[TMP10:%.*]] = and i1 [[TMP5]], [[TMP9]] +; RECOV-NEXT: br i1 [[TMP10]], label [[ASAN_REPORT:%.*]], label [[TMP11:%.*]], !prof [[PROF0]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_load4_noabort(i64 [[TMP0]]) #[[ATTR3]] -; RECOV-NEXT: br label [[TMP12]] -; RECOV: 12: -; RECOV-NEXT: br label [[TMP13]] -; RECOV: 13: +; RECOV-NEXT: br label [[TMP11]] +; RECOV: 11: ; RECOV-NEXT: [[Q:%.*]] = load i32, ptr addrspace(1) [[P]], align 4 ; RECOV-NEXT: ret void ; @@ -124,11 +130,18 @@ define protected amdgpu_kernel void @global_store_8(ptr addrspace(1) %p) sanitiz ; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; CHECK-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] -; CHECK: 6: -; CHECK-NEXT: call void @__asan_report_store8(i64 [[TMP0]]) #[[ATTR3]] -; CHECK-NEXT: unreachable -; CHECK: 7: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0 +; CHECK-NEXT: br i1 [[TMP7]], label [[ASAN_REPORT:%.*]], label [[TMP10:%.*]], !prof [[PROF0]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP5]], label [[TMP8:%.*]], label [[TMP9:%.*]] +; CHECK: 8: +; CHECK-NEXT: call void @__asan_report_store8(i64 [[TMP0]]) #[[ATTR5]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP9]] +; CHECK: 9: +; CHECK-NEXT: br label [[TMP10]] +; CHECK: 10: ; CHECK-NEXT: store i64 0, ptr addrspace(1) [[P]], align 8 ; CHECK-NEXT: ret void ; @@ -141,11 +154,11 @@ define protected amdgpu_kernel void @global_store_8(ptr addrspace(1) %p) sanitiz ; RECOV-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; RECOV-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; RECOV-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; RECOV-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] -; RECOV: 6: +; RECOV-NEXT: br i1 [[TMP5]], label [[ASAN_REPORT:%.*]], label [[TMP6:%.*]], !prof [[PROF0]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_store8_noabort(i64 [[TMP0]]) #[[ATTR3]] -; RECOV-NEXT: br label [[TMP7]] -; RECOV: 7: +; RECOV-NEXT: br label [[TMP6]] +; RECOV: 6: ; RECOV-NEXT: store i64 0, ptr addrspace(1) [[P]], align 8 ; RECOV-NEXT: ret void ; @@ -164,11 +177,18 @@ define protected amdgpu_kernel void @global_load_8(ptr addrspace(1) %p) sanitize ; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; CHECK-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] -; CHECK: 6: -; CHECK-NEXT: call void @__asan_report_load8(i64 [[TMP0]]) #[[ATTR3]] -; CHECK-NEXT: unreachable -; CHECK: 7: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 0 +; CHECK-NEXT: br i1 [[TMP7]], label [[ASAN_REPORT:%.*]], label [[TMP10:%.*]], !prof [[PROF0]] +; CHECK: asan.report: +; CHECK-NEXT: br i1 [[TMP5]], label [[TMP8:%.*]], label [[TMP9:%.*]] +; CHECK: 8: +; CHECK-NEXT: call void @__asan_report_load8(i64 [[TMP0]]) #[[ATTR5]] +; CHECK-NEXT: call void @llvm.amdgcn.unreachable() +; CHECK-NEXT: br label [[TMP9]] +; CHECK: 9: +; CHECK-NEXT: br label [[TMP10]] +; CHECK: 10: ; CHECK-NEXT: [[Q:%.*]] = load i64, ptr addrspace(1) [[P]], align 8 ; CHECK-NEXT: ret void ; @@ -181,11 +201,11 @@ define protected amdgpu_kernel void @global_load_8(ptr addrspace(1) %p) sanitize ; RECOV-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr ; RECOV-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1 ; RECOV-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0 -; RECOV-NEXT: br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]] -; RECOV: 6: +; RECOV-NEXT: br i1 [[TMP5]], label [[ASAN_REPORT:%.*]], label [[TMP6:%.*]], !prof [[PROF0]] +; RECOV: asan.report: ; RECOV-NEXT: call void @__asan_report_load8_noabort(i64 [[TMP0]]) #[[ATTR3]] -; RECOV-NEXT: br label [[TMP7]] -; RECOV: 7: +; RECOV-NEXT: br label [[TMP6]] +; RECOV: 6: ; RECOV-NEXT: [[Q:%.*]] = load i64, ptr addrspace(1) [[P]], align 8 ; RECOV-NEXT: ret void ; -- GitLab From 4241e847072cb436bde0abc74f2ef446a01f29aa Mon Sep 17 00:00:00 2001 From: Aart Bik <39774503+aartbik@users.noreply.github.com> Date: Thu, 4 Jan 2024 14:09:31 -0800 Subject: [PATCH 048/728] [mlir][sparse] minor comment edits in sparsifier pipeline (#77000) --- .../SparseTensor/Pipelines/SparseTensorPipelines.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp index ec74c1c2d54a..66362d5be4f1 100644 --- a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp +++ b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp @@ -32,7 +32,6 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm, const SparsifierOptions &options) { // Rewrite named linalg ops into generic ops. - pm.addNestedPass(createLinalgGeneralizationPass()); // Sparsification and bufferization mini-pipeline. @@ -65,6 +64,8 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm, pm.addNestedPass(createConvertGpuOpsToNVVMOps()); } + // Progressively lower to LLVM. Note that the convert-vector-to-llvm + // pass is repeated on purpose. // TODO(springerm): Add sparse support to the BufferDeallocation pass and add // it to this pipeline. pm.addNestedPass(createConvertLinalgToLoopsPass()); @@ -80,10 +81,7 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm, pm.addNestedPass(createConvertMathToLLVMPass()); pm.addPass(createConvertMathToLibmPass()); pm.addPass(createConvertComplexToLibmPass()); - - // Repeat convert-vector-to-llvm. pm.addPass(createConvertVectorToLLVMPass(options.lowerVectorToLLVMOptions())); - pm.addPass(createConvertComplexToLLVMPass()); pm.addPass(createConvertVectorToLLVMPass(options.lowerVectorToLLVMOptions())); pm.addPass(createConvertFuncToLLVMPass()); @@ -101,6 +99,7 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm, pm.addPass(createGpuModuleToBinaryPass(gpuModuleToBinaryPassOptions)); } + // Ensure all casts are realized. pm.addPass(createReconcileUnrealizedCastsPass()); } -- GitLab From 0414cf09f02239254c449b7c5a37428fecb943f7 Mon Sep 17 00:00:00 2001 From: ZijunZhaoCCK <88353225+ZijunZhaoCCK@users.noreply.github.com> Date: Thu, 4 Jan 2024 14:25:13 -0800 Subject: [PATCH 049/728] Move nondiscard tests of ranges::contains() to the right place. (#76887) nondiscard tests of ranges::contains() should be in ranges.nodiscard_extensions files rather than nodiscard_extensions files --- .../diagnostics/nodiscard_extensions.compile.pass.cpp | 3 --- .../test/libcxx/diagnostics/nodiscard_extensions.verify.cpp | 5 ----- .../ranges.nodiscard_extensions.compile.pass.cpp | 6 ++++++ .../diagnostics/ranges.nodiscard_extensions.verify.cpp | 4 ++++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.compile.pass.cpp b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.compile.pass.cpp index 641fcd9233bc..e9fab0c75a98 100644 --- a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.compile.pass.cpp +++ b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.compile.pass.cpp @@ -45,9 +45,6 @@ void test_algorithms() { #if TEST_STD_VER >= 17 std::clamp(2, 1, 3); std::clamp(2, 3, 1, std::greater()); -#endif -#if TEST_STD_VER >= 23 - std::ranges::contains(arr, arr + 1, 1); #endif std::count_if(std::begin(arr), std::end(arr), P()); std::count(std::begin(arr), std::end(arr), 1); diff --git a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.verify.cpp b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.verify.cpp index 1e3f537f01ed..d7a26d99e522 100644 --- a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.verify.cpp +++ b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.verify.cpp @@ -60,11 +60,6 @@ void test_algorithms() { std::clamp(2, 1, 3, std::greater()); #endif -#if TEST_STD_VER >= 23 - // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} - std::ranges::contains(arr, arr + 1, 1); -#endif - // expected-warning@+1 {{ignoring return value of function declared with 'nodiscard' attribute}} std::count_if(std::begin(arr), std::end(arr), P()); diff --git a/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.compile.pass.cpp b/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.compile.pass.cpp index 7e2d64b8c9b6..19e07b83079c 100644 --- a/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.compile.pass.cpp +++ b/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.compile.pass.cpp @@ -15,6 +15,8 @@ #include +#include "test_macros.h" + void test() { int range[1]; int* iter = range; @@ -28,6 +30,10 @@ void test() { std::ranges::binary_search(range, 1); std::ranges::binary_search(iter, iter, 1); std::ranges::clamp(1, 2, 3); +#if TEST_STD_VER >= 23 + std::ranges::contains(range, 1); + std::ranges::contains(iter, iter, 1); +#endif std::ranges::count_if(range, pred); std::ranges::count_if(iter, iter, pred); std::ranges::count(range, 1); diff --git a/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.verify.cpp b/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.verify.cpp index f0a0e4889a76..5e45ad086cbd 100644 --- a/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.verify.cpp +++ b/libcxx/test/libcxx/diagnostics/ranges.nodiscard_extensions.verify.cpp @@ -91,6 +91,10 @@ void test() { std::ranges::upper_bound(iter, iter, 1); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} #if TEST_STD_VER >= 23 + std::ranges::contains(range, 1); + // expected-warning@-1{{ignoring return value of function declared with 'nodiscard' attribute}} + std::ranges::contains(iter, iter, 1); + // expected-warning@-1{{ignoring return value of function declared with 'nodiscard' attribute}} std::ranges::fold_left(range, 0, std::plus()); // expected-warning@-1{{ignoring return value of function declared with 'nodiscard' attribute}} std::ranges::fold_left(iter, iter, 0, std::plus()); -- GitLab From 241fe83704476f81e3438e32b6d988ea123e624d Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 4 Jan 2024 22:53:18 +0000 Subject: [PATCH 050/728] [VPlan] Introduce ComputeReductionResult VPInstruction opcode. (#70253) This patch introduces a new ComputeReductionResult opcode to compute the final reduction result in the middle block. The code from fixReduction has been moved to ComputeReductionResult, after some earlier cleanup changes to model parts of fixReduction explicitly elsewhere as needed. The recipe may be broken down further in the future. Note that the phi nodes to merge the reduction result from the trip count check and the middle block, to be used as resume value for the scalar remainder loop are also generated based on ComputeReductionResult. Once we have a VPValue for the reduction result, this can also be modeled explicitly and moved out of the recipe. --- .../Vectorize/LoopVectorizationPlanner.h | 16 +- .../Transforms/Vectorize/LoopVectorize.cpp | 351 +++++++----------- llvm/lib/Transforms/Vectorize/VPlan.cpp | 1 + llvm/lib/Transforms/Vectorize/VPlan.h | 5 +- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 82 ++++ .../AArch64/scalable-strict-fadd.ll | 2 +- .../LoopVectorize/AArch64/strict-fadd.ll | 2 +- .../AArch64/sve-interleaved-accesses.ll | 2 +- .../ARM/tail-fold-multiple-icmps.ll | 2 +- .../epilog-vectorization-reductions.ll | 4 +- ...-order-recurrence-sink-replicate-region.ll | 3 +- .../LoopVectorize/interleaved-accesses.ll | 2 +- .../reduction-with-invariant-store.ll | 16 +- .../LoopVectorize/vplan-printing.ll | 7 +- 14 files changed, 247 insertions(+), 248 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 150ab301e7fd..cff72ae263d8 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -346,16 +346,20 @@ public: /// Return the best VPlan for \p VF. VPlan &getBestPlanFor(ElementCount VF) const; - /// Generate the IR code for the body of the vectorized loop according to the - /// best selected \p VF, \p UF and VPlan \p BestPlan. + /// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan + /// according to the best selected \p VF and \p UF. + /// /// TODO: \p IsEpilogueVectorization is needed to avoid issues due to epilogue /// vectorization re-using plans for both the main and epilogue vector loops. /// It should be removed once the re-use issue has been fixed. /// \p ExpandedSCEVs is passed during execution of the plan for epilogue loop - /// to re-use expansion results generated during main plan execution. Returns - /// a mapping of SCEVs to their expanded IR values. Note that this is a - /// temporary workaround needed due to the current epilogue handling. - DenseMap + /// to re-use expansion results generated during main plan execution. + /// + /// Returns a mapping of SCEVs to their expanded IR values and a mapping for + /// the reduction resume values. Note that this is a temporary workaround + /// needed due to the current epilogue handling. + std::pair, + DenseMap> executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool IsEpilogueVectorization, diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 961d3d3bb193..10c068e3b589 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -584,10 +584,6 @@ public: /// able to vectorize with strict in-order reductions for the given RdxDesc. bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc); - // Returns the resume value (bc.merge.rdx) for a reduction as - // generated by fixReduction. - PHINode *getReductionResumeValue(const RecurrenceDescriptor &RdxDesc); - /// Create a new phi node for the induction variable \p OrigPhi to resume /// iteration count in the scalar epilogue, from where the vectorized loop /// left off. \p Step is the SCEV-expanded induction step to use. In cases @@ -626,9 +622,6 @@ protected: BasicBlock *MiddleBlock, BasicBlock *VectorHeader, VPlan &Plan, VPTransformState &State); - /// Handle all cross-iteration phis in the header. - void fixCrossIterationPHIs(VPTransformState &State); - /// Create the exit value of first order recurrences in the middle block and /// update their users. void fixFixedOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR, @@ -1166,14 +1159,6 @@ void InnerLoopVectorizer::collectPoisonGeneratingRecipes( } } -PHINode *InnerLoopVectorizer::getReductionResumeValue( - const RecurrenceDescriptor &RdxDesc) { - auto It = ReductionResumeValues.find(&RdxDesc); - assert(It != ReductionResumeValues.end() && - "Expected to find a resume value for the reduction."); - return It->second; -} - namespace llvm { // Loop vectorization cost-model hints how the scalar epilogue loop should be @@ -3434,8 +3419,15 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State, // At this point every instruction in the original loop is widened to a // vector form. Now we need to fix the recurrences in the loop. These PHI // nodes are currently empty because we did not want to introduce cycles. - // This is the second stage of vectorizing recurrences. - fixCrossIterationPHIs(State); + // This is the second stage of vectorizing recurrences. Note that fixing + // reduction phis are already modeled in VPlan. + // TODO: Also model fixing fixed-order recurrence phis in VPlan. + VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion(); + VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock(); + for (VPRecipeBase &R : HeaderVPBB->phis()) { + if (auto *FOR = dyn_cast(&R)) + fixFixedOrderRecurrence(FOR, State); + } // Forget the original basic block. PSE.getSE()->forgetLoop(OrigLoop); @@ -3450,7 +3442,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State, for (PHINode &PN : Exit->phis()) PSE.getSE()->forgetLcssaPhiWithNewPredecessor(OrigLoop, &PN); - VPBasicBlock *LatchVPBB = Plan.getVectorLoopRegion()->getExitingBasicBlock(); + VPBasicBlock *LatchVPBB = VectorRegion->getExitingBasicBlock(); Loop *VectorLoop = LI->getLoopFor(State.CFG.VPBB2IRBB[LatchVPBB]); if (Cost->requiresScalarEpilogue(VF.isVector())) { // No edge from the middle block to the unique exit block has been inserted @@ -3503,27 +3495,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State, VF.getKnownMinValue() * UF); } -void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) { - // In order to support recurrences we need to be able to vectorize Phi nodes. - // Phi nodes have cycles, so we need to vectorize them in two stages. This is - // stage #2: We now need to fix the recurrences by adding incoming edges to - // the currently empty PHI nodes. At this point every instruction in the - // original loop is widened to a vector form so we can use them to construct - // the incoming edges. - VPBasicBlock *Header = - State.Plan->getVectorLoopRegion()->getEntryBasicBlock(); - - for (VPRecipeBase &R : Header->phis()) { - if (auto *ReductionPhi = dyn_cast(&R)) - fixReduction(ReductionPhi, State); - } - - for (VPRecipeBase &R : Header->phis()) { - if (auto *FOR = dyn_cast(&R)) - fixFixedOrderRecurrence(FOR, State); - } -} - void InnerLoopVectorizer::fixFixedOrderRecurrence( VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State) { // This is the second phase of vectorizing first-order recurrences. An @@ -3643,169 +3614,6 @@ void InnerLoopVectorizer::fixFixedOrderRecurrence( Phi->setName("scalar.recur"); } -void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR, - VPTransformState &State) { - PHINode *OrigPhi = cast(PhiR->getUnderlyingValue()); - // Get it's reduction variable descriptor. - assert(Legal->isReductionVariable(OrigPhi) && - "Unable to find the reduction variable"); - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - - RecurKind RK = RdxDesc.getRecurrenceKind(); - TrackingVH ReductionStartValue = RdxDesc.getRecurrenceStartValue(); - Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); - if (auto *I = dyn_cast(&*ReductionStartValue)) - State.setDebugLocFrom(I->getDebugLoc()); - - VPValue *LoopExitInstDef = PhiR->getBackedgeValue(); - - // Before each round, move the insertion point right between - // the PHIs and the values we are going to write. - // This allows us to write both PHINodes and the extractelement - // instructions. - Builder.SetInsertPoint(LoopMiddleBlock, - LoopMiddleBlock->getFirstInsertionPt()); - - State.setDebugLocFrom(LoopExitInst->getDebugLoc()); - - Type *PhiTy = OrigPhi->getType(); - // If tail is folded by masking, the vector value to leave the loop should be - // a Select choosing between the vectorized LoopExitInst and vectorized Phi, - // instead of the former. For an inloop reduction the reduction will already - // be predicated, and does not need to be handled here. - if (Cost->foldTailByMasking() && !PhiR->isInLoop()) { - VPValue *Def = nullptr; - for (VPUser *U : LoopExitInstDef->users()) { - auto *S = dyn_cast(U); - if (S && S->getOpcode() == Instruction::Select) { - Def = S; - break; - } - } - if (Def) - LoopExitInstDef = Def; - } - - VectorParts RdxParts(UF); - for (unsigned Part = 0; Part < UF; ++Part) - RdxParts[Part] = State.get(LoopExitInstDef, Part); - - // If the vector reduction can be performed in a smaller type, we truncate - // then extend the loop exit value to enable InstCombine to evaluate the - // entire expression in the smaller type. - if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) { - Builder.SetInsertPoint(LoopMiddleBlock, - LoopMiddleBlock->getFirstInsertionPt()); - Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF); - for (unsigned Part = 0; Part < UF; ++Part) { - RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy); - } - } - - // Reduce all of the unrolled parts into a single vector. - Value *ReducedPartRdx = RdxParts[0]; - unsigned Op = RecurrenceDescriptor::getOpcode(RK); - - // The middle block terminator has already been assigned a DebugLoc here (the - // OrigLoop's single latch terminator). We want the whole middle block to - // appear to execute on this line because: (a) it is all compiler generated, - // (b) these instructions are always executed after evaluating the latch - // conditional branch, and (c) other passes may add new predecessors which - // terminate on this line. This is the easiest way to ensure we don't - // accidentally cause an extra step back into the loop while debugging. - State.setDebugLocFrom(LoopMiddleBlock->getTerminator()->getDebugLoc()); - if (PhiR->isOrdered()) - ReducedPartRdx = RdxParts[UF - 1]; - else { - // Floating-point operations should have some FMF to enable the reduction. - IRBuilderBase::FastMathFlagGuard FMFG(Builder); - Builder.setFastMathFlags(RdxDesc.getFastMathFlags()); - for (unsigned Part = 1; Part < UF; ++Part) { - Value *RdxPart = RdxParts[Part]; - if (Op != Instruction::ICmp && Op != Instruction::FCmp) - ReducedPartRdx = Builder.CreateBinOp( - (Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, "bin.rdx"); - else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) - ReducedPartRdx = createAnyOfOp(Builder, ReductionStartValue, RK, - ReducedPartRdx, RdxPart); - else - ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart); - } - } - - // Create the reduction after the loop. Note that inloop reductions create the - // target reduction in the loop using a Reduction recipe. - if (VF.isVector() && !PhiR->isInLoop()) { - ReducedPartRdx = - createTargetReduction(Builder, RdxDesc, ReducedPartRdx, OrigPhi); - // If the reduction can be performed in a smaller type, we need to extend - // the reduction to the wider type before we branch to the original loop. - if (PhiTy != RdxDesc.getRecurrenceType()) - ReducedPartRdx = RdxDesc.isSigned() - ? Builder.CreateSExt(ReducedPartRdx, PhiTy) - : Builder.CreateZExt(ReducedPartRdx, PhiTy); - } - - PHINode *ResumePhi = - dyn_cast(PhiR->getStartValue()->getUnderlyingValue()); - - // Create a phi node that merges control-flow from the backedge-taken check - // block and the middle block. - PHINode *BCBlockPhi = PHINode::Create(PhiTy, 2, "bc.merge.rdx", - LoopScalarPreHeader->getTerminator()); - - // If we are fixing reductions in the epilogue loop then we should already - // have created a bc.merge.rdx Phi after the main vector body. Ensure that - // we carry over the incoming values correctly. - for (auto *Incoming : predecessors(LoopScalarPreHeader)) { - if (Incoming == LoopMiddleBlock) - BCBlockPhi->addIncoming(ReducedPartRdx, Incoming); - else if (ResumePhi && llvm::is_contained(ResumePhi->blocks(), Incoming)) - BCBlockPhi->addIncoming(ResumePhi->getIncomingValueForBlock(Incoming), - Incoming); - else - BCBlockPhi->addIncoming(ReductionStartValue, Incoming); - } - - // Set the resume value for this reduction - ReductionResumeValues.insert({&RdxDesc, BCBlockPhi}); - - // If there were stores of the reduction value to a uniform memory address - // inside the loop, create the final store here. - if (StoreInst *SI = RdxDesc.IntermediateStore) { - StoreInst *NewSI = - Builder.CreateAlignedStore(ReducedPartRdx, SI->getPointerOperand(), - SI->getAlign()); - propagateMetadata(NewSI, SI); - - // If the reduction value is used in other places, - // then let the code below create PHI's for that. - } - - // Now, we need to fix the users of the reduction variable - // inside and outside of the scalar remainder loop. - - // We know that the loop is in LCSSA form. We need to update the PHI nodes - // in the exit blocks. See comment on analogous loop in - // fixFixedOrderRecurrence for a more complete explaination of the logic. - if (!Cost->requiresScalarEpilogue(VF.isVector())) - for (PHINode &LCSSAPhi : LoopExitBlock->phis()) - if (llvm::is_contained(LCSSAPhi.incoming_values(), LoopExitInst)) { - LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock); - State.Plan->removeLiveOut(&LCSSAPhi); - } - - // Fix the scalar loop reduction variable with the incoming reduction sum - // from the vector body and from the backedge value. - int IncomingEdgeBlockIdx = - OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch()); - assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index"); - // Pick the other block. - int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1); - OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi); - OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst); -} - void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) { // The basic block and loop containing the predicated instruction. auto *PredBB = PredInst->getParent(); @@ -7609,7 +7417,65 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) { } } -SCEV2ValueTy LoopVectorizationPlanner::executePlan( +// Check if \p RedResult is a ComputeReductionResult instruction, and if it is +// create a merge phi node for it and add it to \p ReductionResumeValues. +static void createAndCollectMergePhiForReduction( + VPInstruction *RedResult, + DenseMap &ReductionResumeValues, + VPTransformState &State, Loop *OrigLoop, BasicBlock *LoopMiddleBlock) { + if (!RedResult || + RedResult->getOpcode() != VPInstruction::ComputeReductionResult) + return; + + auto *PhiR = cast(RedResult->getOperand(0)); + const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); + + TrackingVH ReductionStartValue = RdxDesc.getRecurrenceStartValue(); + Value *FinalValue = + State.get(RedResult, VPIteration(State.UF - 1, VPLane::getFirstLane())); + auto *ResumePhi = + dyn_cast(PhiR->getStartValue()->getUnderlyingValue()); + + // TODO: bc.merge.rdx should not be created here, instead it should be + // modeled in VPlan. + BasicBlock *LoopScalarPreHeader = OrigLoop->getLoopPreheader(); + // Create a phi node that merges control-flow from the backedge-taken check + // block and the middle block. + auto *BCBlockPhi = PHINode::Create(FinalValue->getType(), 2, "bc.merge.rdx", + LoopScalarPreHeader->getTerminator()); + + // If we are fixing reductions in the epilogue loop then we should already + // have created a bc.merge.rdx Phi after the main vector body. Ensure that + // we carry over the incoming values correctly. + for (auto *Incoming : predecessors(LoopScalarPreHeader)) { + if (Incoming == LoopMiddleBlock) + BCBlockPhi->addIncoming(FinalValue, Incoming); + else if (ResumePhi && is_contained(ResumePhi->blocks(), Incoming)) + BCBlockPhi->addIncoming(ResumePhi->getIncomingValueForBlock(Incoming), + Incoming); + else + BCBlockPhi->addIncoming(ReductionStartValue, Incoming); + } + + auto *OrigPhi = cast(PhiR->getUnderlyingValue()); + // TODO: This fixup should instead be modeled in VPlan. + // Fix the scalar loop reduction variable with the incoming reduction sum + // from the vector body and from the backedge value. + int IncomingEdgeBlockIdx = + OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch()); + assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index"); + // Pick the other block. + int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1); + OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi); + Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); + OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst); + + ReductionResumeValues[&RdxDesc] = BCBlockPhi; +} + +std::pair, + DenseMap> +LoopVectorizationPlanner::executePlan( ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan, InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueVectorization, const DenseMap *ExpandedSCEVs) { @@ -7688,6 +7554,17 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan( BestVPlan.execute(&State); + // 2.5 Collect reduction resume values. + DenseMap ReductionResumeValues; + auto *ExitVPBB = + cast(BestVPlan.getVectorLoopRegion()->getSingleSuccessor()); + for (VPRecipeBase &R : *ExitVPBB) { + createAndCollectMergePhiForReduction(dyn_cast(&R), + ReductionResumeValues, State, OrigLoop, + State.CFG.VPBB2IRBB[ExitVPBB]); + } + + // 2.6. Maintain Loop Hints // Keep all loop hints from the original loop on the vector loop (we'll // replace the vectorizer-specific hints below). MDNode *OrigLoopID = OrigLoop->getLoopID(); @@ -7721,7 +7598,7 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan( ILV.printDebugTracesAtEnd(); - return State.ExpandedSCEVs; + return {State.ExpandedSCEVs, ReductionResumeValues}; } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -9001,10 +8878,15 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) { // to reductions, with one operand being vector and the other being the scalar // reduction chain. For other reductions, a select is introduced between the phi // and live-out recipes when folding the tail. +// +// A ComputeReductionResult recipe is added to the middle block, also for +// in-loop reductions which compute their result in-loop, because generating +// the subsequent bc.merge.rdx phi is driven by ComputeReductionResult recipes. void LoopVectorizationPlanner::adjustRecipesForReductions( VPBasicBlock *LatchVPBB, VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) { - VPBasicBlock *Header = Plan->getVectorLoopRegion()->getEntryBasicBlock(); + VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion(); + VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock(); // Gather all VPReductionPHIRecipe and sort them so that Intermediate stores // sank outside of the loop would keep the same order as they had in the // original loop. @@ -9044,15 +8926,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( for (VPRecipeBase *R : ReductionPHIList) R->moveBefore(*Header, Header->getFirstNonPhi()); - SmallVector InLoopReductionPhis; for (VPRecipeBase &R : Header->phis()) { auto *PhiR = dyn_cast(&R); if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered())) continue; - InLoopReductionPhis.push_back(PhiR); - } - for (VPReductionPHIRecipe *PhiR : InLoopReductionPhis) { const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); RecurKind Kind = RdxDesc.getRecurrenceKind(); assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) && @@ -9161,28 +9039,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( for (VPRecipeBase &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) { VPReductionPHIRecipe *PhiR = dyn_cast(&R); - if (!PhiR || PhiR->isInLoop()) + if (!PhiR) continue; const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - auto *NewExitingVPV = PhiR->getBackedgeValue(); // If tail is folded by masking, introduce selects between the phi // and the live-out instruction of each reduction, at the beginning of the // dedicated latch block. - if (CM.foldTailByMasking()) { + auto *OrigExitingVPV = PhiR->getBackedgeValue(); + auto *NewExitingVPV = PhiR->getBackedgeValue(); + if (!PhiR->isInLoop() && CM.foldTailByMasking()) { VPValue *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), *Plan); - VPValue *Red = PhiR->getBackedgeValue(); - assert(Red->getDefiningRecipe()->getParent() != LatchVPBB && + assert(OrigExitingVPV->getDefiningRecipe()->getParent() != LatchVPBB && "reduction recipe must be defined before latch"); Type *PhiTy = PhiR->getOperand(0)->getLiveInIRValue()->getType(); std::optional FMFs = PhiTy->isFloatingPointTy() ? std::make_optional(RdxDesc.getFastMathFlags()) : std::nullopt; - NewExitingVPV = Builder.createSelect(Cond, Red, PhiR, {}, "", FMFs); - Red->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) { - return isa(&U); + NewExitingVPV = + Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", FMFs); + OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) { + return isa(&U) && + cast(&U)->getOpcode() == + VPInstruction::ComputeReductionResult; }); if (PreferPredicatedReductionSelect || TTI.preferPredicatedReductionSelect( @@ -9190,6 +9071,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( TargetTransformInfo::ReductionFlags())) PhiR->setOperand(1, NewExitingVPV); } + // If the vector reduction can be performed in a smaller type, we truncate // then extend the loop exit value to enable InstCombine to evaluate the // entire expression in the smaller type. @@ -9206,9 +9088,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( Trunc->insertAfter(NewExitingVPV->getDefiningRecipe()); Extnd->insertAfter(Trunc); - NewExitingVPV->replaceAllUsesWith(Extnd); - Trunc->setOperand(0, NewExitingVPV); + if (PhiR->getOperand(1) == NewExitingVPV) + PhiR->setOperand(1, Extnd->getVPSingleValue()); + NewExitingVPV = Extnd; } + + // We want code in the middle block to appear to execute on the location of + // the scalar loop's latch terminator because: (a) it is all compiler + // generated, (b) these instructions are always executed after evaluating + // the latch conditional branch, and (c) other passes may add new + // predecessors which terminate on this line. This is the easiest way to + // ensure we don't accidentally cause an extra step back into the loop while + // debugging. + DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc(); + + // TODO: At the moment ComputeReductionResult also drives creation of the + // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here + // even for in-loop reductions, until the reduction resume value handling is + // also modeled in VPlan. + auto *FinalReductionResult = new VPInstruction( + VPInstruction::ComputeReductionResult, {PhiR, NewExitingVPV}, ExitDL); + cast(VectorLoopRegion->getSingleSuccessor()) + ->appendRecipe(FinalReductionResult); + OrigExitingVPV->replaceUsesWithIf( + FinalReductionResult, + [](VPUser &User, unsigned) { return isa(&User); }); } VPlanTransforms::clearReductionWrapFlags(*Plan); @@ -10175,8 +10079,8 @@ bool LoopVectorizePass::processLoop(Loop *L) { EPI, &LVL, &CM, BFI, PSI, Checks); VPlan &BestMainPlan = LVP.getBestPlanFor(EPI.MainLoopVF); - auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF, - BestMainPlan, MainILV, DT, true); + const auto &[ExpandedSCEVs, ReductionResumeValues] = LVP.executePlan( + EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan, MainILV, DT, true); ++LoopsVectorized; // Second pass vectorizes the epilogue and adjusts the control flow @@ -10217,8 +10121,9 @@ bool LoopVectorizePass::processLoop(Loop *L) { Value *ResumeV = nullptr; // TODO: Move setting of resume values to prepareToExecute. if (auto *ReductionPhi = dyn_cast(&R)) { - ResumeV = MainILV.getReductionResumeValue( - ReductionPhi->getRecurrenceDescriptor()); + ResumeV = ReductionResumeValues + .find(&ReductionPhi->getRecurrenceDescriptor()) + ->second; } else { // Create induction resume values for both widened pointer and // integer/fp inductions and update the start value of the induction diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 1d7df9c9575a..b6e56c47c227 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -446,6 +446,7 @@ void VPBasicBlock::execute(VPTransformState *State) { // ExitBB can be re-used for the exit block of the Plan. NewBB = State->CFG.ExitBB; State->CFG.PrevBB = NewBB; + State->Builder.SetInsertPoint(NewBB->getFirstNonPHI()); // Update the branch instruction in the predecessor to branch to ExitBB. VPBlockBase *PredVPB = getSingleHierarchicalPredecessor(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 7d33baac52c9..9d279da758ec 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1061,7 +1061,8 @@ public: // Increment the canonical IV separately for each unrolled part. CanonicalIVIncrementForPart, BranchOnCount, - BranchOnCond + BranchOnCond, + ComputeReductionResult, }; private: @@ -3132,6 +3133,8 @@ inline bool isUniformAfterVectorization(VPValue *VPV) { return Rep->isUniform(); if (auto *GEP = dyn_cast(Def)) return all_of(GEP->operands(), isUniformAfterVectorization); + if (auto *VPI = dyn_cast(Def)) + return VPI->getOpcode() == VPInstruction::ComputeReductionResult; return false; } } // end namespace vputils diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 76961629aece..1e5273bcd748 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -28,6 +28,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/LoopUtils.h" #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" #include @@ -401,6 +402,84 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, Builder.GetInsertBlock()->getTerminator()->eraseFromParent(); return CondBr; } + case VPInstruction::ComputeReductionResult: { + if (Part != 0) + return State.get(this, 0); + + // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary + // and will be removed by breaking up the recipe further. + auto *PhiR = cast(getOperand(0)); + auto *OrigPhi = cast(PhiR->getUnderlyingValue()); + // Get its reduction variable descriptor. + const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); + + RecurKind RK = RdxDesc.getRecurrenceKind(); + + State.setDebugLocFrom(getDebugLoc()); + + VPValue *LoopExitingDef = getOperand(1); + Type *PhiTy = OrigPhi->getType(); + VectorParts RdxParts(State.UF); + for (unsigned Part = 0; Part < State.UF; ++Part) + RdxParts[Part] = State.get(LoopExitingDef, Part); + + // If the vector reduction can be performed in a smaller type, we truncate + // then extend the loop exit value to enable InstCombine to evaluate the + // entire expression in the smaller type. + // TODO: Handle this in truncateToMinBW. + if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) { + Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF); + for (unsigned Part = 0; Part < State.UF; ++Part) + RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy); + } + // Reduce all of the unrolled parts into a single vector. + Value *ReducedPartRdx = RdxParts[0]; + unsigned Op = RecurrenceDescriptor::getOpcode(RK); + + if (PhiR->isOrdered()) { + ReducedPartRdx = RdxParts[State.UF - 1]; + } else { + // Floating-point operations should have some FMF to enable the reduction. + IRBuilderBase::FastMathFlagGuard FMFG(Builder); + Builder.setFastMathFlags(RdxDesc.getFastMathFlags()); + for (unsigned Part = 1; Part < State.UF; ++Part) { + Value *RdxPart = RdxParts[Part]; + if (Op != Instruction::ICmp && Op != Instruction::FCmp) + ReducedPartRdx = Builder.CreateBinOp( + (Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, "bin.rdx"); + else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) { + TrackingVH ReductionStartValue = + RdxDesc.getRecurrenceStartValue(); + ReducedPartRdx = createAnyOfOp(Builder, ReductionStartValue, RK, + ReducedPartRdx, RdxPart); + } else + ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart); + } + } + + // Create the reduction after the loop. Note that inloop reductions create + // the target reduction in the loop using a Reduction recipe. + if (State.VF.isVector() && !PhiR->isInLoop()) { + ReducedPartRdx = + createTargetReduction(Builder, RdxDesc, ReducedPartRdx, OrigPhi); + // If the reduction can be performed in a smaller type, we need to extend + // the reduction to the wider type before we branch to the original loop. + if (PhiTy != RdxDesc.getRecurrenceType()) + ReducedPartRdx = RdxDesc.isSigned() + ? Builder.CreateSExt(ReducedPartRdx, PhiTy) + : Builder.CreateZExt(ReducedPartRdx, PhiTy); + } + + // If there were stores of the reduction value to a uniform memory address + // inside the loop, create the final store here. + if (StoreInst *SI = RdxDesc.IntermediateStore) { + auto *NewSI = Builder.CreateAlignedStore( + ReducedPartRdx, SI->getPointerOperand(), SI->getAlign()); + propagateMetadata(NewSI, SI); + } + + return ReducedPartRdx; + } default: llvm_unreachable("Unsupported opcode for instruction"); } @@ -477,6 +556,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent, case VPInstruction::BranchOnCount: O << "branch-on-count"; break; + case VPInstruction::ComputeReductionResult: + O << "compute-reduction-result"; + break; default: O << Instruction::getOpcodeName(getOpcode()); } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index acbb6e86fe58..e51190bae612 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -612,8 +612,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-UNORDERED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK-UNORDERED: middle.block: -; CHECK-UNORDERED-NEXT: [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP14]]) ; CHECK-UNORDERED-NEXT: [[TMP20:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP15]]) +; CHECK-UNORDERED-NEXT: [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP14]]) ; CHECK-UNORDERED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK-UNORDERED: scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll index 33b5273217cd..7c1247e9ebc8 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll @@ -285,8 +285,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <4 x float> %[[STRIDED2:.*]], %[[VEC_PHI2]] ; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd ; CHECK-UNORDERED: middle.block -; CHECK-UNORDERED: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD1]]) ; CHECK-UNORDERED: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD2]]) +; CHECK-UNORDERED: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD1]]) ; CHECK-UNORDERED: for.body ; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, ptr ; CHECK-UNORDERED: %[[FADD1:.*]] = fadd float %[[LOAD1]], {{.*}} diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll index f600a0d5877d..fa8e35f4697e 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll @@ -821,8 +821,8 @@ define void @int_float_struct(%struct.IntFloat* nocapture readonly %p) #0 { ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP4]]) ; CHECK-NEXT: [[TMP10:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP5]]) +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP4]]) ; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: br label [[FOR_BODY:%.*]] diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll index 038725db279b..f58d864e1e14 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-fold-multiple-icmps.ll @@ -26,8 +26,8 @@ define arm_aapcs_vfpcc i32 @minmaxval4(ptr nocapture readonly %x, ptr nocapture ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]]) ; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP2]]) ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[N]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[SCALAR_PH]] ; CHECK: scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll index 03903d80cfd6..4df5332a47d4 100644 --- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll @@ -313,8 +313,8 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP3]]) ; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP3]]) ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]] ; CHECK: vec.epilog.iter.check: @@ -344,8 +344,8 @@ define float @multiple_fp_rdx(ptr %A, i64 %N) { ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]] ; CHECK-NEXT: br i1 [[TMP15]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: vec.epilog.middle.block: -; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP13]]) ; CHECK-NEXT: [[TMP17:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP14]]) +; CHECK-NEXT: [[TMP16:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP13]]) ; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]] ; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]] ; CHECK: vec.epilog.scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index 060d28f030f7..833d55f09294 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -215,9 +215,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]> ; CHECK-NEXT: No successors ; CHECK-EMPTY: -; CHECK-NEXT: Live-out i32 %res = vp<[[SEL]]> +; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]> ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll index 187eefbe9b59..3b9edb11c5da 100644 --- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll @@ -837,8 +837,8 @@ define void @int_float_struct(ptr nocapture readonly %A) #0 { ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]]) ; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]]) ; CHECK-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: br label [[FOR_BODY:%.*]] diff --git a/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll b/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll index 2a2d55fb75c5..5584aa969367 100644 --- a/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll +++ b/llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll @@ -562,10 +562,10 @@ exit: ; preds = %for.body define void @reduc_add_mul_store_same_ptr(ptr %dst, ptr readonly %src) { ; CHECK-LABEL: define void @reduc_add_mul_store_same_ptr ; CHECK: middle.block: -; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]]) -; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4 ; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP3:%.*]]) ; CHECK-NEXT: store i32 [[TMP4]], ptr %dst, align 4 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]]) +; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4 ; entry: br label %for.body @@ -591,10 +591,10 @@ exit: define void @reduc_mul_add_store_same_ptr(ptr %dst, ptr readonly %src) { ; CHECK-LABEL: define void @reduc_mul_add_store_same_ptr ; CHECK: middle.block: -; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]]) -; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4 ; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3:%.*]]) ; CHECK-NEXT: store i32 [[TMP4]], ptr %dst, align 4 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]]) +; CHECK-NEXT: store i32 [[TMP2]], ptr %dst, align 4 ; entry: br label %for.body @@ -621,10 +621,10 @@ exit: define void @reduc_add_mul_store_different_ptr(ptr %dst1, ptr %dst2, ptr readonly %src) { ; CHECK-LABEL: define void @reduc_add_mul_store_different_ptr ; CHECK: middle.block: -; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]]) -; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4 ; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP3:%.*]]) ; CHECK-NEXT: store i32 [[TMP4]], ptr %dst2, align 4 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1:%.*]]) +; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4 ; entry: br label %for.body @@ -650,10 +650,10 @@ exit: define void @reduc_mul_add_store_different_ptr(ptr %dst1, ptr %dst2, ptr readonly %src) { ; CHECK-LABEL: define void @reduc_mul_add_store_different_ptr ; CHECK: middle.block: -; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]]) -; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4 ; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3:%.*]]) ; CHECK-NEXT: store i32 [[TMP4]], ptr %dst2, align 4 +; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP1:%.*]]) +; CHECK-NEXT: store i32 [[TMP2]], ptr %dst1, align 4 ; entry: br label %for.body diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 52a18d7b365c..79ed71a2685f 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -137,9 +137,10 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%red>, ir<%red.next> ; CHECK-NEXT: No successors ; CHECK-EMPTY: -; CHECK-NEXT: Live-out float %red.next.lcssa = ir<%red.next> +; CHECK-NEXT: Live-out float %red.next.lcssa = vp<[[RED_RES]]> ; CHECK-NEXT: } ; entry: @@ -185,6 +186,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result ir<%red>, ir<%red.next> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -385,9 +387,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%sum.07>, ir<[[MULADD]]> ; CHECK-NEXT: No successors ; CHECK-EMPTY: -; CHECK-NEXT: Live-out float %muladd.lcssa = ir<%muladd> +; CHECK-NEXT: Live-out float %muladd.lcssa = vp<[[RED_RES]]> ; CHECK-NEXT:} entry: -- GitLab From 665e46c2689cc4212345213db7d7e968b91dcc8b Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Thu, 4 Jan 2024 15:03:18 -0800 Subject: [PATCH 051/728] [llvm-profdata] Use semicolon as the delimiter for supplementary profiles. (#75080) When merging instrFDO profiles with afdo profile as supplementary, instrFDO counters for static functions are stored with function's PGO name (with filename.cpp; prefix). - This pull request fixes the delimiter used when a PGO function name is 'normalized' for AFDO look-up. --- .../profile/Linux/instrprof-instr-suppl.test | 86 +++++++++++++++++++ .../llvm-profdata/Inputs/FUnique.proftext | 2 +- .../llvm-profdata/Inputs/NoFUnique.proftext | 2 +- .../Inputs/flatten_instr.proftext | 2 +- .../suppl-instr-with-sample-flatten.test | 6 +- llvm/tools/llvm-profdata/llvm-profdata.cpp | 17 ++-- 6 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 compiler-rt/test/profile/Linux/instrprof-instr-suppl.test diff --git a/compiler-rt/test/profile/Linux/instrprof-instr-suppl.test b/compiler-rt/test/profile/Linux/instrprof-instr-suppl.test new file mode 100644 index 000000000000..10650a345ab4 --- /dev/null +++ b/compiler-rt/test/profile/Linux/instrprof-instr-suppl.test @@ -0,0 +1,86 @@ +// This is a regression test for supplementary profiles. + +// What the test does: +// - Generate raw profiles from an executable and convert it to indexed profiles. +// - Merge indexed profiles with supplementary sample-pgo profiles +// - Check that the block counters for function foo is scaled up. + +// REQUIRES: lld-available + +// Building the instrumented binary will fail because lld doesn't support +// big-endian ELF for PPC (aka ABI 1). +// ld.lld: error: /lib/../lib64/Scrt1.o: ABI version 1 is not supported +// UNSUPPORTED: ppc && host-byteorder-big-endian + +// This compiler-rt test aims to have test coverage for the IRPGO name format +// of local-linkage functions during raw profile generation. The C++ functions +// are simple with little optimization space so test outputs are more stable. +// On the other hand, LLVM tests (like tools/llvm-profdata/suppl-instr-with-sample-static-func.test +// or other suppl* test under tools/llvm-profdata dir) are more suitable for +// more sophisticated cases (e.g., pseudo hot functions or profiles with discriminiators, etc). + +// RUN: rm -rf %t && split-file %s %t && cd %t + +// Use clangxx_pgogen for IR level instrumentation for C++. +// The test case is constructed such that `-funique-internal-linkage-names` is +// not used in instrPGO but used in static function names in SamplePGO. +// RUN: %clangxx_pgogen -fuse-ld=lld -O2 main.cpp -o main +// RUN: env LLVM_PROFILE_FILE=main.profraw %run ./main +// RUN: llvm-profdata merge main.profraw -o main.profdata + +// The function counters are not scaled up. +// RUN: llvm-profdata show -all-functions -counts main.profdata | FileCheck %s --check-prefix=INSTR + +// The instrPGO profile counter of function foo should be scaled up. Note the +// scaling factor of a function is computed based on instrPGO profiles and +// invariant to samplePGO profile counters. +// RUN: llvm-profdata merge -supplement-instr-with-sample=sampleprof.proftext \ +// RUN: -suppl-min-size-threshold=0 -instr-prof-cold-threshold=1 \ +// RUN: main.profdata -o merge.profdata +// RUN: llvm-profdata show -all-functions -counts merge.profdata | FileCheck %s --check-prefix=SUPPL + +// INSTR: Counters: +// INSTR: main: +// INSTR: Counters: 1 +// INSTR: Block counts: [1] +// INSTR: _Z3barv: +// INSTR: Counters: 1 +// INSTR: Block counts: [2] +// INSTR: main.cpp;_ZL3foov: +// INSTR: Counters: 1 +// INSTR: Block counts: [1] + +// INSTR: Functions shown: 3 +// INSTR: Total functions: 3 + +// SUPPL: Counters: +// SUPPL: main: +// SUPPL: Counters: 1 +// SUPPL: Block counts: [1] +// SUPPL: _Z3barv: +// SUPPL: Counters: 1 +// SUPPL: Block counts: [2] +// SUPPL: main.cpp;_ZL3foov: +// SUPPL: Counters: 1 +// SUPPL: Block counts: [3] + +//--- main.cpp + +// mark foo and bar as noinline so preinliner won't inlined them into main +// before the instrumentation pass. +__attribute__((noinline)) static void foo() { +} + +__attribute__((noinline)) void bar() { +} + +int main() { + foo(); + bar(); + bar(); + return 0; +} + +//--- sampleprof.proftext +_ZL3foov.__uniq.23343505234642233139497840575431302970:5:5 + 1: 5 diff --git a/llvm/test/tools/llvm-profdata/Inputs/FUnique.proftext b/llvm/test/tools/llvm-profdata/Inputs/FUnique.proftext index be2c27df9746..da169b18201a 100644 --- a/llvm/test/tools/llvm-profdata/Inputs/FUnique.proftext +++ b/llvm/test/tools/llvm-profdata/Inputs/FUnique.proftext @@ -18,7 +18,7 @@ main 1 0 -test.c:_ZL3foom.__uniq.276699478366846449772231447066107882794 +test.c;_ZL3foom.__uniq.276699478366846449772231447066107882794 # Func Hash: 1124680652115249575 # Num Counters: diff --git a/llvm/test/tools/llvm-profdata/Inputs/NoFUnique.proftext b/llvm/test/tools/llvm-profdata/Inputs/NoFUnique.proftext index 7485a33875e1..a3df42f6c3d2 100644 --- a/llvm/test/tools/llvm-profdata/Inputs/NoFUnique.proftext +++ b/llvm/test/tools/llvm-profdata/Inputs/NoFUnique.proftext @@ -18,7 +18,7 @@ main 1 0 -test.c:_ZL3foom +test.c;_ZL3foom # Func Hash: 1124680652115249575 # Num Counters: diff --git a/llvm/test/tools/llvm-profdata/Inputs/flatten_instr.proftext b/llvm/test/tools/llvm-profdata/Inputs/flatten_instr.proftext index 8afee8f67b3b..e18009973151 100644 --- a/llvm/test/tools/llvm-profdata/Inputs/flatten_instr.proftext +++ b/llvm/test/tools/llvm-profdata/Inputs/flatten_instr.proftext @@ -14,7 +14,7 @@ foo 40 6000 -bar.cc:bar +bar.cc;bar # Func Hash: 2222 # Num Counters: diff --git a/llvm/test/tools/llvm-profdata/suppl-instr-with-sample-flatten.test b/llvm/test/tools/llvm-profdata/suppl-instr-with-sample-flatten.test index 2c569b5cf3b3..4a394d7bce5c 100644 --- a/llvm/test/tools/llvm-profdata/suppl-instr-with-sample-flatten.test +++ b/llvm/test/tools/llvm-profdata/suppl-instr-with-sample-flatten.test @@ -7,11 +7,11 @@ RUN: -supplement-instr-with-sample=%p/Inputs/flatten_sample.proftext \ RUN: %p/Inputs/flatten_instr.proftext -o %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s --check-prefix=FLATTEN -FLATTEN: bar.cc:bar: -FLATTEN-NEXT: Hash: 0x00000000000008ae -FLATTEN-NEXT: Counters: 10 FLATTEN: foo: FLATTEN-NEXT: Hash: 0x0000000000000457 FLATTEN-NEXT: Counters: 5 FLATTEN-NEXT: Block counts: [10000, 50, 2000, 40, 6000] +FLATTEN: bar.cc;bar: +FLATTEN-NEXT: Hash: 0x00000000000008ae +FLATTEN-NEXT: Counters: 10 FLATTEN-NOT: goo: diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 12b81d411cfa..77197d3bf919 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -998,13 +998,14 @@ adjustInstrProfile(std::unique_ptr &WC, auto buildStaticFuncMap = [&StaticFuncMap, SampleProfileHasFUnique](const StringRef Name) { - std::string Prefixes[] = {".cpp:", "cc:", ".c:", ".hpp:", ".h:"}; + std::string FilePrefixes[] = {".cpp", "cc", ".c", ".hpp", ".h"}; size_t PrefixPos = StringRef::npos; - for (auto &Prefix : Prefixes) { - PrefixPos = Name.find_insensitive(Prefix); + for (auto &FilePrefix : FilePrefixes) { + std::string NamePrefix = FilePrefix + kGlobalIdentifierDelimiter; + PrefixPos = Name.find_insensitive(NamePrefix); if (PrefixPos == StringRef::npos) continue; - PrefixPos += Prefix.size(); + PrefixPos += NamePrefix.size(); break; } @@ -1088,17 +1089,17 @@ adjustInstrProfile(std::unique_ptr &WC, // // InstrProfile has two entries: // foo - // bar.cc:bar + // bar.cc;bar // // After BuildMaxSampleMap, we should have the following in FlattenSampleMap: // {"foo", {1000, 5000}} - // {"bar.cc:bar", {11000, 30000}} + // {"bar.cc;bar", {11000, 30000}} // // foo's has an entry count of 1000, and max body count of 5000. - // bar.cc:bar has an entry count of 11000 (sum two callsites of 1000 and + // bar.cc;bar has an entry count of 11000 (sum two callsites of 1000 and // 10000), and max count of 30000 (from the callsite in line 8). // - // Note that goo's count will remain in bar.cc:bar() as it does not have an + // Note that goo's count will remain in bar.cc;bar() as it does not have an // entry in InstrProfile. llvm::StringMap> FlattenSampleMap; auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap, -- GitLab From 786cf76f434d2691878067dedb8b1eb99472810b Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Thu, 4 Jan 2024 15:30:23 -0800 Subject: [PATCH 052/728] [dsymutil] Add support for inline DWARF source files. (#77016) There was a strange and seemingly unncessary empty string optimization in NonRelocatableStringPool that I had to remove in order to support empty strings in the line_str string table, without unconditionally forcing an empty string to be added to every debug_line_str table. --- .../llvm/CodeGen/NonRelocatableStringpool.h | 3 +- llvm/lib/CodeGen/NonRelocatableStringpool.cpp | 3 -- llvm/lib/DWARFLinker/DWARFStreamer.cpp | 16 +++++++-- .../tools/dsymutil/ARM/inline-source.test | 20 +++++++++++ llvm/test/tools/dsymutil/Inputs/inline.ll | 34 +++++++++++++++++++ 5 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 llvm/test/tools/dsymutil/ARM/inline-source.test create mode 100644 llvm/test/tools/dsymutil/Inputs/inline.ll diff --git a/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h b/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h index fe07c70d85c5..3dc0731f5a04 100644 --- a/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h +++ b/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h @@ -32,7 +32,7 @@ public: bool PutEmptyString = false) : Translator(Translator) { if (PutEmptyString) - EmptyString = getEntry(""); + getEntry(""); } DwarfStringPoolEntryRef getEntry(StringRef S); @@ -59,7 +59,6 @@ private: MapTy Strings; uint64_t CurrentEndOffset = 0; unsigned NumEntries = 0; - DwarfStringPoolEntryRef EmptyString; std::function Translator; }; diff --git a/llvm/lib/CodeGen/NonRelocatableStringpool.cpp b/llvm/lib/CodeGen/NonRelocatableStringpool.cpp index 7304bfef55cb..e8391afb8e3f 100644 --- a/llvm/lib/CodeGen/NonRelocatableStringpool.cpp +++ b/llvm/lib/CodeGen/NonRelocatableStringpool.cpp @@ -12,9 +12,6 @@ namespace llvm { DwarfStringPoolEntryRef NonRelocatableStringpool::getEntry(StringRef S) { - if (S.empty() && !Strings.empty()) - return EmptyString; - if (Translator) S = Translator(S); auto I = Strings.insert({S, DwarfStringPoolEntry()}); diff --git a/llvm/lib/DWARFLinker/DWARFStreamer.cpp b/llvm/lib/DWARFLinker/DWARFStreamer.cpp index cd649c328ed9..faa91364ec9c 100644 --- a/llvm/lib/DWARFLinker/DWARFStreamer.cpp +++ b/llvm/lib/DWARFLinker/DWARFStreamer.cpp @@ -859,21 +859,31 @@ void DwarfStreamer::emitLineTablePrologueV5IncludeAndFileTable( for (auto Include : P.IncludeDirectories) emitLineTableString(P, Include, DebugStrPool, DebugLineStrPool); + bool InlineSources = any_of(P.FileNames, [](auto &File) { + auto s = dwarf::toString(File.Source); + return s && !**s; + }); + if (P.FileNames.empty()) { // file_name_entry_format_count (ubyte). MS->emitInt8(0); LineSectionSize += 1; } else { // file_name_entry_format_count (ubyte). - MS->emitInt8(2); + MS->emitInt8(2 + (InlineSources ? 1 : 0)); LineSectionSize += 1; // file_name_entry_format (sequence of ULEB128 pairs). + auto StrForm = P.FileNames[0].Name.getForm(); LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_LNCT_path); - LineSectionSize += MS->emitULEB128IntValue(P.FileNames[0].Name.getForm()); + LineSectionSize += MS->emitULEB128IntValue(StrForm); LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_LNCT_directory_index); LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_FORM_data1); + if (InlineSources) { + LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_LNCT_LLVM_source); + LineSectionSize += MS->emitULEB128IntValue(StrForm); + } } // file_names_count (ULEB128). @@ -884,6 +894,8 @@ void DwarfStreamer::emitLineTablePrologueV5IncludeAndFileTable( emitLineTableString(P, File.Name, DebugStrPool, DebugLineStrPool); MS->emitInt8(File.DirIdx); LineSectionSize += 1; + if (InlineSources) + emitLineTableString(P, File.Source, DebugStrPool, DebugLineStrPool); } } diff --git a/llvm/test/tools/dsymutil/ARM/inline-source.test b/llvm/test/tools/dsymutil/ARM/inline-source.test new file mode 100644 index 000000000000..ec437e3de900 --- /dev/null +++ b/llvm/test/tools/dsymutil/ARM/inline-source.test @@ -0,0 +1,20 @@ +# RUN: rm -rf %t +# RUN: mkdir -p %t +# RUN: llc -filetype=obj -mtriple arm64-apple-darwin %p/../Inputs/inline.ll -o %t/inline.o +# RUN: dsymutil -f -oso-prepend-path=%t -y %s -o - | llvm-dwarfdump -debug-line - | FileCheck %s + +# Test inline source files. + +--- +triple: 'arm64-apple-darwin' +objects: + - filename: inline.o + symbols: + - { sym: _f, objAddr: 0x0, binAddr: 0x1000, size: 0x12 } +... + +# CHECK: .debug_line contents: +# CHECK: file_names[ 1]: +# CHECK-NEXT: name: "inlined.c" +# CHECK-NEXT: dir_index: 1 +# CHECK-NEXT: source: "{{.*}}This is inline source code. \ No newline at end of file diff --git a/llvm/test/tools/dsymutil/Inputs/inline.ll b/llvm/test/tools/dsymutil/Inputs/inline.ll new file mode 100644 index 000000000000..e7c9b7096ee5 --- /dev/null +++ b/llvm/test/tools/dsymutil/Inputs/inline.ll @@ -0,0 +1,34 @@ +; ModuleID = '/tmp/t.c' +source_filename = "/tmp/t.c" +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" + +; Function Attrs: noinline nounwind optnone ssp uwtable(sync) +define void @f() #0 !dbg !9 { +entry: + ret void, !dbg !14 +} + +attributes #0 = { noinline nounwind optnone ssp uwtable(sync) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3, !4, !5, !6, !7} +!llvm.ident = !{!8} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0git (git@github.com:llvm/llvm-project.git 29ee66f4a0967e43a035f147c960743c7b640f2f)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: Apple, sysroot: "/") +!1 = !DIFile(filename: "/INLINE/inlined.c", directory: "/Volumes/Data/llvm-project", checksumkind: CSK_MD5, checksum: "3183154a5cb31debe9a8e27ca500bc3c") +!2 = !{i32 7, !"Dwarf Version", i32 5} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !{i32 1, !"wchar_size", i32 4} +!5 = !{i32 8, !"PIC Level", i32 2} +!6 = !{i32 7, !"uwtable", i32 1} +!7 = !{i32 7, !"frame-pointer", i32 1} +!8 = !{!"clang version 18.0.0git (git@github.com:llvm/llvm-project.git 29ee66f4a0967e43a035f147c960743c7b640f2f)"} +!9 = distinct !DISubprogram(name: "f", scope: !10, file: !10, line: 2, type: !11, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0) +!10 = !DIFile(filename: "/INLINE/inlined.c", directory: "", source: "void stop(); +void f() { + // This is inline source code. +} +") +!11 = !DISubroutineType(types: !12) +!12 = !{null} +!14 = !DILocation(line: 4, column: 1, scope: !9) -- GitLab From 9a2df55f47e4ec02a1efbf8efa776cfeed527df2 Mon Sep 17 00:00:00 2001 From: Ellis Hoag Date: Thu, 4 Jan 2024 16:13:57 -0800 Subject: [PATCH 053/728] [InstrProf] No linkage prefixes in IRPGO names (#76994) Change the format of IRPGO counter names to `[;]` which is computed by `GlobalValue::getGlobalIdentifier()` to fix #74565. In fe051934cbb0aaf25d960d7d45305135635d650b (https://reviews.llvm.org/D156569) the format of IRPGO counter names was changed to be `[;]` where `` is basically `F.getName()` with some prefix, e.g., `_` or `l_` on Mach-O (yes, it is confusing that `` is computed with `Mangler().getNameWithPrefix()` while `` is just `F.getName()`). We discovered in #74565 that this causes some missed import issues on some targets and #74008 is a partial fix. Since `` may not match the `` on some targets like Mach-O, we will need to post-process the output of `llvm-profdata order` before passing to the linker via `-order_file`. Profiles generated after fe051934cbb0aaf25d960d7d45305135635d650b will become stale after this diff, but I think this is acceptable since that patch landed after the LLVM 18 cut which hasn't been released yet. --- ...trprof-thinlto-indirect-call-promotion.cpp | 14 ---------- lld/test/MachO/pgo-warn-mismatch.ll | 2 +- llvm/lib/ProfileData/InstrProf.cpp | 27 ++++++------------- llvm/tools/llvm-profdata/llvm-profdata.cpp | 6 ++++- llvm/unittests/ProfileData/InstrProfTest.cpp | 18 ++----------- 5 files changed, 16 insertions(+), 51 deletions(-) diff --git a/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp b/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp index 860c054f69e1..d361186cf26a 100644 --- a/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp +++ b/compiler-rt/test/profile/instrprof-thinlto-indirect-call-promotion.cpp @@ -28,20 +28,6 @@ // LTO if default linker is GNU ld or gold anyway. // REQUIRES: lld-available -// Test should fail where linkage-name and mangled-name diverges, see issue https://github.com/llvm/llvm-project/issues/74565). -// Currently, this name divergence happens on Mach-O object file format, or on -// many (but not all) 32-bit Windows systems. -// -// XFAIL: system-darwin -// -// Mark 32-bit Windows as UNSUPPORTED for now as opposed to XFAIL. This test -// should fail on many (but not all) 32-bit Windows systems and succeed on the -// rest. The flexibility in triple string parsing makes it tricky to capture -// both sets accurately. i[3-9]86 specifies arch as Triple::ArchType::x86, (win32|windows) -// specifies OS as Triple::OS::Win32 -// -// UNSUPPORTED: target={{i.86.*windows.*}} - // RUN: rm -rf %t && split-file %s %t && cd %t // Do setup work for all below tests. diff --git a/lld/test/MachO/pgo-warn-mismatch.ll b/lld/test/MachO/pgo-warn-mismatch.ll index 632adffb01fb..365eeaccb4b4 100644 --- a/lld/test/MachO/pgo-warn-mismatch.ll +++ b/lld/test/MachO/pgo-warn-mismatch.ll @@ -24,7 +24,7 @@ entry: ;--- cs.proftext :csir -_foo +foo # Func Hash: 2277602155505015273 # Num Counters: diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 134a400e639c..4264da8ad751 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -14,7 +14,6 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" @@ -27,7 +26,6 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" -#include "llvm/IR/Mangler.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" @@ -297,29 +295,20 @@ static StringRef getStrippedSourceFileName(const GlobalObject &GO) { return FileName; } -// The PGO name has the format [;] where ; is -// provided if linkage is local and is the mangled function -// name. The filepath is used to discriminate possibly identical function names. -// ; is used because it is unlikely to be found in either or -// . +// The PGO name has the format [;] where ; is +// provided if linkage is local and is used to discriminate possibly identical +// mangled names. ";" is used because it is unlikely to be found in either +// or . // // Older compilers used getPGOFuncName() which has the format -// [:]. is used to discriminate between -// possibly identical function names when linkage is local and -// simply comes from F.getName(). This caused trouble for Objective-C functions -// which commonly have :'s in their names. Also, since is not -// mangled, they cannot be passed to Mach-O linkers via -order_file. We still -// need to compute this name to lookup functions from profiles built by older -// compilers. +// [:]. This caused trouble for Objective-C functions +// which commonly have :'s in their names. We still need to compute this name to +// lookup functions from profiles built by older compilers. static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName) { - SmallString<64> Name; - // FIXME: Mangler's handling is kept outside of `getGlobalIdentifier` for now. - // For more details please check issue #74565. - Mangler().getNameWithPrefix(Name, &GO, /*CannotUsePrivateLabel=*/true); - return GlobalValue::getGlobalIdentifier(Name, Linkage, FileName); + return GlobalValue::getGlobalIdentifier(GO.getName(), Linkage, FileName); } static std::optional lookupPGONameFromMetadata(MDNode *MD) { diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp index 77197d3bf919..05e96f48cf12 100644 --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -3158,7 +3158,11 @@ static int order_main(int argc, const char *argv[]) { BalancedPartitioning BP(Config); BP.run(Nodes); - WithColor::note() << "# Ordered " << Nodes.size() << " functions\n"; + OS << "# Ordered " << Nodes.size() << " functions\n"; + OS << "# Warning: Mach-O may prefix symbols with \"_\" depending on the " + "linkage and this output does not take that into account. Some " + "post-processing may be required before passing to the linker via " + "-order_file.\n"; for (auto &N : Nodes) { auto [Filename, ParsedFuncName] = getParsedIRPGOFuncName(Reader->getSymtab().getFuncOrVarName(N.Id)); diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp index 6a71a975fbb1..8ffb68de7a2d 100644 --- a/llvm/unittests/ProfileData/InstrProfTest.cpp +++ b/llvm/unittests/ProfileData/InstrProfTest.cpp @@ -542,22 +542,14 @@ TEST_F(InstrProfTest, test_memprof_merge) { TEST_F(InstrProfTest, test_irpgo_function_name) { LLVMContext Ctx; auto M = std::make_unique("MyModule.cpp", Ctx); - // Use Mach-O mangling so that non-private symbols get a `_` prefix. - M->setDataLayout(DataLayout("m:o")); auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); std::vector> Data; - Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "_ExternalFoo"); + Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo"); Data.emplace_back("InternalFoo", Function::InternalLinkage, - "MyModule.cpp;_InternalFoo"); - Data.emplace_back("PrivateFoo", Function::PrivateLinkage, - "MyModule.cpp;l_PrivateFoo"); - Data.emplace_back("WeakODRFoo", Function::WeakODRLinkage, "_WeakODRFoo"); - // Test Objective-C symbols + "MyModule.cpp;InternalFoo"); Data.emplace_back("\01-[C dynamicFoo:]", Function::ExternalLinkage, "-[C dynamicFoo:]"); - Data.emplace_back("-", Function::ExternalLinkage, - "_-"); Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage, "MyModule.cpp;-[C internalFoo:]"); @@ -590,10 +582,6 @@ TEST_F(InstrProfTest, test_pgo_function_name) { Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo"); Data.emplace_back("InternalFoo", Function::InternalLinkage, "MyModule.cpp:InternalFoo"); - Data.emplace_back("PrivateFoo", Function::PrivateLinkage, - "MyModule.cpp:PrivateFoo"); - Data.emplace_back("WeakODRFoo", Function::WeakODRLinkage, "WeakODRFoo"); - // Test Objective-C symbols Data.emplace_back("\01-[C externalFoo:]", Function::ExternalLinkage, "-[C externalFoo:]"); Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage, @@ -611,8 +599,6 @@ TEST_F(InstrProfTest, test_pgo_function_name) { TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) { LLVMContext Ctx; auto M = std::make_unique("MyModule.cpp", Ctx); - // Use Mach-O mangling so that non-private symbols get a `_` prefix. - M->setDataLayout(DataLayout("m:o")); auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false); auto *InternalFooF = Function::Create(FTy, Function::InternalLinkage, "InternalFoo", M.get()); -- GitLab From 3096353477a6802de9d4c74018c28f13e8ce1310 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Thu, 4 Jan 2024 16:33:06 -0800 Subject: [PATCH 054/728] test-release.sh: Add a CMake cache file for 3-stage release builds (#75903) You can now pass the -use-cmake-cache option to test-release.sh and it will use a predefined cache file for building the release. This will make it easier to reproduce the builds and add other enhancements like PGO or bolt optimizations. --------- Co-authored-by: Konrad Kleine --- .github/workflows/release-binaries.yml | 5 +- clang/cmake/caches/Release.cmake | 41 ++++++++++++++++ llvm/utils/release/test-release.sh | 67 +++++++++++++++++++++++--- 3 files changed, 104 insertions(+), 9 deletions(-) create mode 100644 clang/cmake/caches/Release.cmake diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 4e3eaff97a87..4a4ba9dcc5ca 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -93,8 +93,8 @@ jobs: - name: Build Clang run: | - cmake -G Ninja -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_BUILD_TYPE=Release -DCMAKE_ENABLE_ASSERTIONS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DLLVM_ENABLE_PROJECTS=clang -S llvm -B build - ninja -v -C build + cmake -G Ninja -C clang/cmake/caches/Release.cmake -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_POSITION_INDEPENDENT_CODE=ON -S llvm -B build + ninja -v -C build clang build-binaries: @@ -152,6 +152,7 @@ jobs: -triple ${{ matrix.target.triple }} \ -use-ninja \ -no-checkout \ + -use-cmake-cache \ -no-test-suite \ -configure-flags "-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake new file mode 100644 index 000000000000..3ea65ce26296 --- /dev/null +++ b/clang/cmake/caches/Release.cmake @@ -0,0 +1,41 @@ +# Plain options configure the first build. +# BOOTSTRAP_* options configure the second build. +# BOOTSTRAP_BOOTSTRAP_* options configure the third build. + +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") + +# Stage 1 Bootstrap Setup +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(CLANG_BOOTSTRAP_TARGETS + clang + check-all + check-llvm + check-clang + test-suite + stage3 + stage3-clang + stage3-check-all + stage3-check-llvm + stage3-check-clang + stage3-install + stage3-test-suite CACHE STRING "") + +# Stage 1 Options +set(LLVM_ENABLE_PROJECTS "clang" CACHE STRING "") +set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "") + +# Stage 2 Bootstrap Setup +set(BOOTSTRAP_CLANG_ENABLE_BOOTSTRAP ON CACHE STRING "") +set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS + clang + check-all + check-llvm + check-clang CACHE STRING "") + +# Stage 2 Options +set(BOOTSTRAP_LLVM_ENABLE_PROJECTS "clang" CACHE STRING "") +set(BOOTSTRAP_LLVM_TARGETS_TO_BUILD Native CACHE STRING "") + +# Stage 3 Options +set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "") +set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_PROJECTS "clang;lld;lldb;clang-tools-extra;bolt;polly;mlir;flang" CACHE STRING "") diff --git a/llvm/utils/release/test-release.sh b/llvm/utils/release/test-release.sh index f38134e3d22d..544d4bfdd799 100755 --- a/llvm/utils/release/test-release.sh +++ b/llvm/utils/release/test-release.sh @@ -46,7 +46,7 @@ BuildDir="`pwd`" ExtraConfigureFlags="" ExportBranch="" git_ref="" - +do_cmake_cache="no" do_bolt="no" if [ "$System" = "Linux" ]; then case $Machine in @@ -87,6 +87,7 @@ function usage() { echo " -no-mlir Disable check-out & build MLIR" echo " -no-flang Disable check-out & build Flang" echo " -silent-log Don't output build logs to stdout" + echo " -use-cmake-cache Build using a CMake cache file" } while [ $# -gt 0 ]; do @@ -200,6 +201,9 @@ while [ $# -gt 0 ]; do -silent-log ) do_silent_log="yes" ;; + -use-cmake-cache | --use-cmake-cache ) + do_cmake_cache="yes" + ;; -help | --help | -h | --h | -\? ) usage exit 0 @@ -328,6 +332,55 @@ Package=$Package-$Triple # Errors to be highlighted at the end are written to this file. echo -n > $LogDir/deferred_errors.log +redir="/dev/stdout" +if [ $do_silent_log == "yes" ]; then + echo "# Silencing build logs because of -silent-log flag..." + redir="/dev/null" +fi + + +function build_with_cmake_cache() { +( + CMakeBuildDir=$BuildDir/build + SrcDir=$BuildDir/llvm-project/ + InstallDir=$BuildDir/install + + rm -rf $CMakeBuildDir + + # FIXME: Would be nice if the commands were echoed to the log file too. + set -x + + env CC="$c_compiler" CXX="$cxx_compiler" \ + cmake -G "$generator" -B $CMakeBuildDir -S $SrcDir/llvm \ + -C $SrcDir/clang/cmake/caches/Release.cmake \ + -DCLANG_BOOTSTRAP_PASSTHROUGH="CMAKE_POSITION_INDEPENDENT_CODE;LLVM_LIT_ARGS" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DLLVM_LIT_ARGS="-j $NumJobs $LitVerbose" \ + $ExtraConfigureFlags + 2>&1 | tee $LogDir/llvm.configure-$Flavor.log + + ${MAKE} $J_ARG $Verbose -C $CMakeBuildDir stage3-check-all \ + 2>&1 | tee $LogDir/llvm.make-$Flavor.log > $redir + + DESTDIR="${InstallDir}" \ + ${MAKE} -C $CMakeBuildDir stage3-install \ + 2>&1 | tee $LogDir/llvm.install-$Flavor.log > $redir + + mkdir -p $BuildDir/Release + pushd $BuildDir/Release + mv $InstallDir/usr/local $Package + if [ "$use_gzip" = "yes" ]; then + tar cf - $Package | gzip -9c > $BuildDir/$Package.tar.gz + else + tar cf - $Package | xz -9ce -T $NumJobs > $BuildDir/$Package.tar.xz + fi + mv $Package $InstallDir/usr/local + popd +) 2>&1 | tee $LogDir/testing.$Release-$RC.log + + exit 0 +} + function deferred_error() { Phase="$1" Flavor="$2" @@ -485,12 +538,6 @@ function build_llvmCore() { fi fi - redir="/dev/stdout" - if [ $do_silent_log == "yes" ]; then - echo "# Silencing build logs because of -silent-log flag..." - redir="/dev/null" - fi - cd $ObjDir echo "# Compiling llvm $Release-$RC $Flavor" echo "# ${MAKE} $J_ARG $Verbose" @@ -600,7 +647,13 @@ if [ $do_test_suite = "yes" ]; then mkdir -p $TestSuiteBuildDir fi +if [ "$do_cmake_cache" = "yes" ]; then + build_with_cmake_cache + exit 0 +fi + ( + Flavors="Release" if [ "$do_debug" = "yes" ]; then Flavors="Debug $Flavors" -- GitLab From 71ec30132b74ba6974cece53f2a00f2749cf95f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Thu, 4 Jan 2024 16:33:20 -0800 Subject: [PATCH 055/728] [mlir][openacc] Add device_type support for data operation (#76126) Following #75864, this patch adds device_type support to the data operation on the async and wait operands and attributes. --- flang/lib/Lower/OpenACC.cpp | 125 ++++++++++++------ flang/test/Lower/OpenACC/acc-data.f90 | 8 +- .../mlir/Dialect/OpenACC/OpenACCOps.td | 47 +++++-- mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp | 43 +++++- mlir/test/Dialect/OpenACC/ops.mlir | 8 +- 5 files changed, 176 insertions(+), 55 deletions(-) diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index ecf70818c4ac..d10e56e5d117 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -1464,6 +1464,24 @@ static void genAsyncClause(Fortran::lower::AbstractConverter &converter, } } +static void +genAsyncClause(Fortran::lower::AbstractConverter &converter, + const Fortran::parser::AccClause::Async *asyncClause, + llvm::SmallVector &async, + llvm::SmallVector &asyncDeviceTypes, + llvm::SmallVector &asyncOnlyDeviceTypes, + mlir::acc::DeviceTypeAttr deviceTypeAttr, + Fortran::lower::StatementContext &stmtCtx) { + const auto &asyncClauseValue = asyncClause->v; + if (asyncClauseValue) { // async has a value. + async.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(*asyncClauseValue), stmtCtx))); + asyncDeviceTypes.push_back(deviceTypeAttr); + } else { + asyncOnlyDeviceTypes.push_back(deviceTypeAttr); + } +} + static mlir::acc::DeviceType getDeviceType(Fortran::parser::AccDeviceTypeExpr::Device device) { switch (device) { @@ -1533,6 +1551,39 @@ static void genWaitClause(Fortran::lower::AbstractConverter &converter, } } +static void +genWaitClause(Fortran::lower::AbstractConverter &converter, + const Fortran::parser::AccClause::Wait *waitClause, + llvm::SmallVector &waitOperands, + llvm::SmallVector &waitOperandsDeviceTypes, + llvm::SmallVector &waitOnlyDeviceTypes, + llvm::SmallVector &waitOperandsSegments, + mlir::Value &waitDevnum, mlir::acc::DeviceTypeAttr deviceTypeAttr, + Fortran::lower::StatementContext &stmtCtx) { + const auto &waitClauseValue = waitClause->v; + if (waitClauseValue) { // wait has a value. + const Fortran::parser::AccWaitArgument &waitArg = *waitClauseValue; + const auto &waitList = + std::get>(waitArg.t); + auto crtWaitOperands = waitOperands.size(); + for (const Fortran::parser::ScalarIntExpr &value : waitList) { + waitOperands.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(value), stmtCtx))); + } + waitOperandsDeviceTypes.push_back(deviceTypeAttr); + waitOperandsSegments.push_back(waitOperands.size() - crtWaitOperands); + + // TODO: move to device_type model. + const auto &waitDevnumValue = + std::get>(waitArg.t); + if (waitDevnumValue) + waitDevnum = fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(*waitDevnumValue), stmtCtx)); + } else { + waitOnlyDeviceTypes.push_back(deviceTypeAttr); + } +} + static mlir::acc::LoopOp createLoopOp(Fortran::lower::AbstractConverter &converter, mlir::Location currentLocation, @@ -1795,6 +1846,7 @@ createComputeOp(Fortran::lower::AbstractConverter &converter, firstprivateOperands; llvm::SmallVector privatizations, firstPrivatizations, reductionRecipes; + mlir::Value waitDevnum; // TODO not yet implemented on compute op. // Self clause has optional values but can be present with // no value as well. When there is no value, the op has an attribute to @@ -1818,31 +1870,14 @@ createComputeOp(Fortran::lower::AbstractConverter &converter, mlir::Location clauseLocation = converter.genLocation(clause.source); if (const auto *asyncClause = std::get_if(&clause.u)) { - const auto &asyncClauseValue = asyncClause->v; - if (asyncClauseValue) { // async has a value. - async.push_back(fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(*asyncClauseValue), stmtCtx))); - asyncDeviceTypes.push_back(crtDeviceTypeAttr); - } else { - asyncOnlyDeviceTypes.push_back(crtDeviceTypeAttr); - } + genAsyncClause(converter, asyncClause, async, asyncDeviceTypes, + asyncOnlyDeviceTypes, crtDeviceTypeAttr, stmtCtx); } else if (const auto *waitClause = std::get_if(&clause.u)) { - const auto &waitClauseValue = waitClause->v; - if (waitClauseValue) { // wait has a value. - const Fortran::parser::AccWaitArgument &waitArg = *waitClauseValue; - const auto &waitList = - std::get>(waitArg.t); - auto crtWaitOperands = waitOperands.size(); - for (const Fortran::parser::ScalarIntExpr &value : waitList) { - waitOperands.push_back(fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(value), stmtCtx))); - } - waitOperandsDeviceTypes.push_back(crtDeviceTypeAttr); - waitOperandsSegments.push_back(waitOperands.size() - crtWaitOperands); - } else { - waitOnlyDeviceTypes.push_back(crtDeviceTypeAttr); - } + genWaitClause(converter, waitClause, waitOperands, + waitOperandsDeviceTypes, waitOnlyDeviceTypes, + waitOperandsSegments, waitDevnum, crtDeviceTypeAttr, + stmtCtx); } else if (const auto *numGangsClause = std::get_if( &clause.u)) { @@ -2126,21 +2161,24 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semanticsContext, Fortran::lower::StatementContext &stmtCtx, const Fortran::parser::AccClauseList &accClauseList) { - mlir::Value ifCond, async, waitDevnum; + mlir::Value ifCond, waitDevnum; llvm::SmallVector attachEntryOperands, createEntryOperands, - copyEntryOperands, copyoutEntryOperands, dataClauseOperands, waitOperands; - - // Async and wait have an optional value but can be present with - // no value as well. When there is no value, the op has an attribute to - // represent the clause. - bool addAsyncAttr = false; - bool addWaitAttr = false; + copyEntryOperands, copyoutEntryOperands, dataClauseOperands, waitOperands, + async; + llvm::SmallVector asyncDeviceTypes, asyncOnlyDeviceTypes, + waitOperandsDeviceTypes, waitOnlyDeviceTypes; + llvm::SmallVector waitOperandsSegments; bool hasDefaultNone = false; bool hasDefaultPresent = false; fir::FirOpBuilder &builder = converter.getFirOpBuilder(); + // device_type attribute is set to `none` until a device_type clause is + // encountered. + auto crtDeviceTypeAttr = mlir::acc::DeviceTypeAttr::get( + builder.getContext(), mlir::acc::DeviceType::None); + // Lower clauses values mapped to operands. // Keep track of each group of operands separately as clauses can appear // more than once. @@ -2221,11 +2259,14 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter, dataClauseOperands.end()); } else if (const auto *asyncClause = std::get_if(&clause.u)) { - genAsyncClause(converter, asyncClause, async, addAsyncAttr, stmtCtx); + genAsyncClause(converter, asyncClause, async, asyncDeviceTypes, + asyncOnlyDeviceTypes, crtDeviceTypeAttr, stmtCtx); } else if (const auto *waitClause = std::get_if(&clause.u)) { - genWaitClause(converter, waitClause, waitOperands, waitDevnum, - addWaitAttr, stmtCtx); + genWaitClause(converter, waitClause, waitOperands, + waitOperandsDeviceTypes, waitOnlyDeviceTypes, + waitOperandsSegments, waitDevnum, crtDeviceTypeAttr, + stmtCtx); } else if(const auto *defaultClause = std::get_if(&clause.u)) { if ((defaultClause->v).v == llvm::acc::DefaultValue::ACC_Default_none) @@ -2239,7 +2280,7 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter, llvm::SmallVector operands; llvm::SmallVector operandSegments; addOperand(operands, operandSegments, ifCond); - addOperand(operands, operandSegments, async); + addOperands(operands, operandSegments, async); addOperand(operands, operandSegments, waitDevnum); addOperands(operands, operandSegments, waitOperands); addOperands(operands, operandSegments, dataClauseOperands); @@ -2250,8 +2291,18 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter, auto dataOp = createRegionOp( builder, currentLocation, eval, operands, operandSegments); - dataOp.setAsyncAttr(addAsyncAttr); - dataOp.setWaitAttr(addWaitAttr); + if (!asyncDeviceTypes.empty()) + dataOp.setAsyncDeviceTypeAttr(builder.getArrayAttr(asyncDeviceTypes)); + if (!asyncOnlyDeviceTypes.empty()) + dataOp.setAsyncOnlyAttr(builder.getArrayAttr(asyncOnlyDeviceTypes)); + if (!waitOperandsDeviceTypes.empty()) + dataOp.setWaitOperandsDeviceTypeAttr( + builder.getArrayAttr(waitOperandsDeviceTypes)); + if (!waitOperandsSegments.empty()) + dataOp.setWaitOperandsSegmentsAttr( + builder.getDenseI32ArrayAttr(waitOperandsSegments)); + if (!waitOnlyDeviceTypes.empty()) + dataOp.setWaitOnlyAttr(builder.getArrayAttr(waitOnlyDeviceTypes)); if (hasDefaultNone) dataOp.setDefaultAttr(mlir::acc::ClauseDefaultValue::None); diff --git a/flang/test/Lower/OpenACC/acc-data.f90 b/flang/test/Lower/OpenACC/acc-data.f90 index a6572e147076..75ffd1fc3fca 100644 --- a/flang/test/Lower/OpenACC/acc-data.f90 +++ b/flang/test/Lower/OpenACC/acc-data.f90 @@ -153,7 +153,7 @@ subroutine acc_data !$acc end data ! CHECK: acc.data dataOperands(%{{.*}}) { -! CHECK: } attributes {asyncAttr} +! CHECK: } attributes {asyncOnly = [#acc.device_type]} !$acc data present(a) async(1) !$acc end data @@ -165,18 +165,18 @@ subroutine acc_data !$acc end data ! CHECK: acc.data dataOperands(%{{.*}}) { -! CHECK: } attributes {waitAttr} +! CHECK: } attributes {waitOnly = [#acc.device_type]} !$acc data present(a) wait(1) !$acc end data -! CHECK: acc.data dataOperands(%{{.*}}) wait(%{{.*}} : i32) { +! CHECK: acc.data dataOperands(%{{.*}}) wait({%{{.*}} : i32}) { ! CHECK: }{{$}} !$acc data present(a) wait(devnum: 0: 1) !$acc end data -! CHECK: acc.data dataOperands(%{{.*}}) wait_devnum(%{{.*}} : i32) wait(%{{.*}} : i32) { +! CHECK: acc.data dataOperands(%{{.*}}) wait_devnum(%{{.*}} : i32) wait({%{{.*}} : i32}) { ! CHECK: }{{$}} !$acc data default(none) diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index 4312bd4de1bd..1dd83e933034 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -1236,13 +1236,16 @@ def OpenACC_DataOp : OpenACC_Op<"data", let arguments = (ins Optional:$ifCond, - Optional:$async, - UnitAttr:$asyncAttr, - Optional:$waitDevnum, - Variadic:$waitOperands, - UnitAttr:$waitAttr, - Variadic:$dataClauseOperands, - OptionalAttr:$defaultAttr); + Variadic:$async, + OptionalAttr:$asyncDeviceType, + OptionalAttr:$asyncOnly, + Optional:$waitDevnum, + Variadic:$waitOperands, + OptionalAttr:$waitOperandsSegments, + OptionalAttr:$waitOperandsDeviceType, + OptionalAttr:$waitOnly, + Variadic:$dataClauseOperands, + OptionalAttr:$defaultAttr); let regions = (region AnyRegion:$region); @@ -1252,15 +1255,41 @@ def OpenACC_DataOp : OpenACC_Op<"data", /// The i-th data operand passed. Value getDataOperand(unsigned i); + + /// Return true if the op has the async attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasAsyncOnly(); + /// Return true if the op has the async attribute for the given device_type. + bool hasAsyncOnly(mlir::acc::DeviceType deviceType); + /// Return the value of the async clause if present. + mlir::Value getAsyncValue(); + /// Return the value of the async clause for the given device_type if + /// present. + mlir::Value getAsyncValue(mlir::acc::DeviceType deviceType); + + /// Return true if the op has the wait attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasWaitOnly(); + /// Return true if the op has the wait attribute for the given device_type. + bool hasWaitOnly(mlir::acc::DeviceType deviceType); + /// Return the values of the wait clause if present. + mlir::Operation::operand_range getWaitValues(); + /// Return the values of the wait clause for the given device_type if + /// present. + mlir::Operation::operand_range + getWaitValues(mlir::acc::DeviceType deviceType); }]; let assemblyFormat = [{ oilist( `if` `(` $ifCond `)` - | `async` `(` $async `:` type($async) `)` + | `async` `(` custom($async, + type($async), $asyncDeviceType) `)` | `dataOperands` `(` $dataClauseOperands `:` type($dataClauseOperands) `)` | `wait_devnum` `(` $waitDevnum `:` type($waitDevnum) `)` - | `wait` `(` $waitOperands `:` type($waitOperands) `)` + | `wait` `(` custom($waitOperands, + type($waitOperands), $waitOperandsDeviceType, + $waitOperandsSegments) `)` ) $region attr-dict-with-keyword }]; diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp index e299b67b10a9..66605ead0529 100644 --- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp +++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp @@ -1417,11 +1417,52 @@ unsigned DataOp::getNumDataOperands() { return getDataClauseOperands().size(); } Value DataOp::getDataOperand(unsigned i) { unsigned numOptional = getIfCond() ? 1 : 0; - numOptional += getAsync() ? 1 : 0; + numOptional += getAsync().size() ? 1 : 0; numOptional += getWaitOperands().size(); return getOperand(numOptional + i); } +bool acc::DataOp::hasAsyncOnly() { + return hasAsyncOnly(mlir::acc::DeviceType::None); +} + +bool acc::DataOp::hasAsyncOnly(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getAsyncOnly()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + +mlir::Value DataOp::getAsyncValue() { + return getAsyncValue(mlir::acc::DeviceType::None); +} + +mlir::Value DataOp::getAsyncValue(mlir::acc::DeviceType deviceType) { + return getValueInDeviceTypeSegment(getAsyncDeviceType(), getAsync(), + deviceType); +} + +bool DataOp::hasWaitOnly() { return hasWaitOnly(mlir::acc::DeviceType::None); } + +bool DataOp::hasWaitOnly(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getWaitOnly()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + +mlir::Operation::operand_range DataOp::getWaitValues() { + return getWaitValues(mlir::acc::DeviceType::None); +} + +mlir::Operation::operand_range +DataOp::getWaitValues(mlir::acc::DeviceType deviceType) { + return getValuesFromSegments(getWaitOperandsDeviceType(), getWaitOperands(), + getWaitOperandsSegments(), deviceType); +} + //===----------------------------------------------------------------------===// // ExitDataOp //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir index 5a95811685f8..52375b1af314 100644 --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -836,11 +836,11 @@ func.func @testdataop(%a: memref, %b: memref, %c: memref) -> () { } attributes { defaultAttr = #acc, wait } %w1 = arith.constant 1 : i64 - acc.data wait(%w1 : i64) { + acc.data wait({%w1 : i64}) { } attributes { defaultAttr = #acc, wait } %wd1 = arith.constant 1 : i64 - acc.data wait_devnum(%wd1 : i64) wait(%w1 : i64) { + acc.data wait_devnum(%wd1 : i64) wait({%w1 : i64}) { } attributes { defaultAttr = #acc, wait } return @@ -951,10 +951,10 @@ func.func @testdataop(%a: memref, %b: memref, %c: memref) -> () { // CHECK: acc.data { // CHECK-NEXT: } attributes {defaultAttr = #acc, wait} -// CHECK: acc.data wait(%{{.*}} : i64) { +// CHECK: acc.data wait({%{{.*}} : i64}) { // CHECK-NEXT: } attributes {defaultAttr = #acc, wait} -// CHECK: acc.data wait_devnum(%{{.*}} : i64) wait(%{{.*}} : i64) { +// CHECK: acc.data wait_devnum(%{{.*}} : i64) wait({%{{.*}} : i64}) { // CHECK-NEXT: } attributes {defaultAttr = #acc, wait} // ----- -- GitLab From e456689fb3d6dd785202cd25f89e9443e5ad7d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Thu, 4 Jan 2024 16:33:33 -0800 Subject: [PATCH 056/728] [mlir][flang][openacc] Support device_type on loop construct (#76892) This is adding support for `device_type` clause representation in the OpenACC MLIR dialect on the acc.loop operation and adjust flang to lower correctly to the new representation. Each "value" that can be impacted by a `device_type` clause is now associated with an array attribute that carry this information. This includes: - `worker` clause information - `gang` clause information - `vector` clause information - `collapse` clause information - `tile` clause information The representation of the `gang` clause information has been updated and all values are now carried in a single operand segment. This segment is then subdivided by `device_type`. Each value in a segment is also associated with a `GangArgType` so it can be differentiated (num/dim/static). This simplify the handling of gang values an limit the number of new attributes needed. When the clause can be associated with the operation without any value (`gang`, `vector`, `worker`). These are represented by a dedicated attributes with device_type information. Extra getter functions are provided to make it easier to retrieve a value based on a device_type. --- flang/lib/Lower/OpenACC.cpp | 183 +++++--- flang/test/Lower/OpenACC/acc-kernels-loop.f90 | 36 +- flang/test/Lower/OpenACC/acc-loop.f90 | 41 +- .../test/Lower/OpenACC/acc-parallel-loop.f90 | 36 +- flang/test/Lower/OpenACC/acc-reduction.f90 | 6 +- flang/test/Lower/OpenACC/acc-serial-loop.f90 | 36 +- .../mlir/Dialect/OpenACC/OpenACCOps.td | 141 +++++- mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp | 427 ++++++++++++++---- mlir/test/Dialect/OpenACC/invalid.mlir | 57 ++- mlir/test/Dialect/OpenACC/ops.mlir | 80 ++-- 10 files changed, 737 insertions(+), 306 deletions(-) diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index d10e56e5d117..d24c369d81be 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -1593,67 +1593,89 @@ createLoopOp(Fortran::lower::AbstractConverter &converter, const Fortran::parser::AccClauseList &accClauseList, bool needEarlyReturnHandling = false) { fir::FirOpBuilder &builder = converter.getFirOpBuilder(); - - mlir::Value workerNum; - mlir::Value vectorNum; - mlir::Value gangNum; - mlir::Value gangDim; - mlir::Value gangStatic; llvm::SmallVector tileOperands, privateOperands, - reductionOperands, cacheOperands; + reductionOperands, cacheOperands, vectorOperands, workerNumOperands, + gangOperands; llvm::SmallVector privatizations, reductionRecipes; - bool hasGang = false, hasVector = false, hasWorker = false; + llvm::SmallVector tileOperandsSegments, gangOperandsSegments; + llvm::SmallVector collapseValues; + + llvm::SmallVector gangArgTypes; + llvm::SmallVector seqDeviceTypes, independentDeviceTypes, + autoDeviceTypes, vectorOperandsDeviceTypes, workerNumOperandsDeviceTypes, + vectorDeviceTypes, workerNumDeviceTypes, tileOperandsDeviceTypes, + collapseDeviceTypes, gangDeviceTypes, gangOperandsDeviceTypes; + + // device_type attribute is set to `none` until a device_type clause is + // encountered. + auto crtDeviceTypeAttr = mlir::acc::DeviceTypeAttr::get( + builder.getContext(), mlir::acc::DeviceType::None); for (const Fortran::parser::AccClause &clause : accClauseList.v) { mlir::Location clauseLocation = converter.genLocation(clause.source); if (const auto *gangClause = std::get_if(&clause.u)) { if (gangClause->v) { + auto crtGangOperands = gangOperands.size(); const Fortran::parser::AccGangArgList &x = *gangClause->v; for (const Fortran::parser::AccGangArg &gangArg : x.v) { if (const auto *num = std::get_if(&gangArg.u)) { - gangNum = fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(num->v), stmtCtx)); + gangOperands.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(num->v), stmtCtx))); + gangArgTypes.push_back(mlir::acc::GangArgTypeAttr::get( + builder.getContext(), mlir::acc::GangArgType::Num)); } else if (const auto *staticArg = std::get_if( &gangArg.u)) { const Fortran::parser::AccSizeExpr &sizeExpr = staticArg->v; if (sizeExpr.v) { - gangStatic = fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(*sizeExpr.v), stmtCtx)); + gangOperands.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(*sizeExpr.v), stmtCtx))); } else { // * was passed as value and will be represented as a special // constant. - gangStatic = builder.createIntegerConstant( - clauseLocation, builder.getIndexType(), starCst); + gangOperands.push_back(builder.createIntegerConstant( + clauseLocation, builder.getIndexType(), starCst)); } + gangArgTypes.push_back(mlir::acc::GangArgTypeAttr::get( + builder.getContext(), mlir::acc::GangArgType::Static)); } else if (const auto *dim = std::get_if( &gangArg.u)) { - gangDim = fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(dim->v), stmtCtx)); + gangOperands.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(dim->v), stmtCtx))); + gangArgTypes.push_back(mlir::acc::GangArgTypeAttr::get( + builder.getContext(), mlir::acc::GangArgType::Dim)); } } + gangOperandsSegments.push_back(gangOperands.size() - crtGangOperands); + gangOperandsDeviceTypes.push_back(crtDeviceTypeAttr); + } else { + gangDeviceTypes.push_back(crtDeviceTypeAttr); } - hasGang = true; } else if (const auto *workerClause = std::get_if(&clause.u)) { if (workerClause->v) { - workerNum = fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(*workerClause->v), stmtCtx)); + workerNumOperands.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(*workerClause->v), stmtCtx))); + workerNumOperandsDeviceTypes.push_back(crtDeviceTypeAttr); + } else { + workerNumDeviceTypes.push_back(crtDeviceTypeAttr); } - hasWorker = true; } else if (const auto *vectorClause = std::get_if(&clause.u)) { if (vectorClause->v) { - vectorNum = fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(*vectorClause->v), stmtCtx)); + vectorOperands.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(*vectorClause->v), stmtCtx))); + vectorOperandsDeviceTypes.push_back(crtDeviceTypeAttr); + } else { + vectorDeviceTypes.push_back(crtDeviceTypeAttr); } - hasVector = true; } else if (const auto *tileClause = std::get_if(&clause.u)) { const Fortran::parser::AccTileExprList &accTileExprList = tileClause->v; + auto crtTileOperands = tileOperands.size(); for (const auto &accTileExpr : accTileExprList.v) { const auto &expr = std::get>( @@ -1669,6 +1691,8 @@ createLoopOp(Fortran::lower::AbstractConverter &converter, tileOperands.push_back(tileStar); } } + tileOperandsDeviceTypes.push_back(crtDeviceTypeAttr); + tileOperandsSegments.push_back(tileOperands.size() - crtTileOperands); } else if (const auto *privateClause = std::get_if( &clause.u)) { @@ -1680,17 +1704,46 @@ createLoopOp(Fortran::lower::AbstractConverter &converter, &clause.u)) { genReductions(reductionClause->v, converter, semanticsContext, stmtCtx, reductionOperands, reductionRecipes); + } else if (std::get_if(&clause.u)) { + seqDeviceTypes.push_back(crtDeviceTypeAttr); + } else if (std::get_if( + &clause.u)) { + independentDeviceTypes.push_back(crtDeviceTypeAttr); + } else if (std::get_if(&clause.u)) { + autoDeviceTypes.push_back(crtDeviceTypeAttr); + } else if (const auto *deviceTypeClause = + std::get_if( + &clause.u)) { + const Fortran::parser::AccDeviceTypeExprList &deviceTypeExprList = + deviceTypeClause->v; + assert(deviceTypeExprList.v.size() == 1 && + "expect only one device_type expr"); + crtDeviceTypeAttr = mlir::acc::DeviceTypeAttr::get( + builder.getContext(), getDeviceType(deviceTypeExprList.v.front().v)); + } else if (const auto *collapseClause = + std::get_if( + &clause.u)) { + const Fortran::parser::AccCollapseArg &arg = collapseClause->v; + const auto &force = std::get(arg.t); + if (force) + TODO(clauseLocation, "OpenACC collapse force modifier"); + const auto &intExpr = + std::get(arg.t); + const auto *expr = Fortran::semantics::GetExpr(intExpr); + const std::optional collapseValue = + Fortran::evaluate::ToInt64(*expr); + assert(collapseValue && "expect integer value for the collapse clause"); + collapseValues.push_back(*collapseValue); + collapseDeviceTypes.push_back(crtDeviceTypeAttr); } } // Prepare the operand segment size attribute and the operands value range. llvm::SmallVector operands; llvm::SmallVector operandSegments; - addOperand(operands, operandSegments, gangNum); - addOperand(operands, operandSegments, gangDim); - addOperand(operands, operandSegments, gangStatic); - addOperand(operands, operandSegments, workerNum); - addOperand(operands, operandSegments, vectorNum); + addOperands(operands, operandSegments, gangOperands); + addOperands(operands, operandSegments, workerNumOperands); + addOperands(operands, operandSegments, vectorOperands); addOperands(operands, operandSegments, tileOperands); addOperands(operands, operandSegments, cacheOperands); addOperands(operands, operandSegments, privateOperands); @@ -1708,12 +1761,42 @@ createLoopOp(Fortran::lower::AbstractConverter &converter, builder, currentLocation, eval, operands, operandSegments, /*outerCombined=*/false, retTy, yieldValue); - if (hasGang) - loopOp.setHasGangAttr(builder.getUnitAttr()); - if (hasWorker) - loopOp.setHasWorkerAttr(builder.getUnitAttr()); - if (hasVector) - loopOp.setHasVectorAttr(builder.getUnitAttr()); + if (!gangDeviceTypes.empty()) + loopOp.setGangAttr(builder.getArrayAttr(gangDeviceTypes)); + if (!gangArgTypes.empty()) + loopOp.setGangOperandsArgTypeAttr(builder.getArrayAttr(gangArgTypes)); + if (!gangOperandsSegments.empty()) + loopOp.setGangOperandsSegmentsAttr( + builder.getDenseI32ArrayAttr(gangOperandsSegments)); + if (!gangOperandsDeviceTypes.empty()) + loopOp.setGangOperandsDeviceTypeAttr( + builder.getArrayAttr(gangOperandsDeviceTypes)); + + if (!workerNumDeviceTypes.empty()) + loopOp.setWorkerAttr(builder.getArrayAttr(workerNumDeviceTypes)); + if (!workerNumOperandsDeviceTypes.empty()) + loopOp.setWorkerNumOperandsDeviceTypeAttr( + builder.getArrayAttr(workerNumOperandsDeviceTypes)); + + if (!vectorDeviceTypes.empty()) + loopOp.setVectorAttr(builder.getArrayAttr(vectorDeviceTypes)); + if (!vectorOperandsDeviceTypes.empty()) + loopOp.setVectorOperandsDeviceTypeAttr( + builder.getArrayAttr(vectorOperandsDeviceTypes)); + + if (!tileOperandsDeviceTypes.empty()) + loopOp.setTileOperandsDeviceTypeAttr( + builder.getArrayAttr(tileOperandsDeviceTypes)); + if (!tileOperandsSegments.empty()) + loopOp.setTileOperandsSegmentsAttr( + builder.getDenseI32ArrayAttr(tileOperandsSegments)); + + if (!seqDeviceTypes.empty()) + loopOp.setSeqAttr(builder.getArrayAttr(seqDeviceTypes)); + if (!independentDeviceTypes.empty()) + loopOp.setIndependentAttr(builder.getArrayAttr(independentDeviceTypes)); + if (!autoDeviceTypes.empty()) + loopOp.setAuto_Attr(builder.getArrayAttr(autoDeviceTypes)); if (!privatizations.empty()) loopOp.setPrivatizationsAttr( @@ -1723,33 +1806,11 @@ createLoopOp(Fortran::lower::AbstractConverter &converter, loopOp.setReductionRecipesAttr( mlir::ArrayAttr::get(builder.getContext(), reductionRecipes)); - // Lower clauses mapped to attributes - for (const Fortran::parser::AccClause &clause : accClauseList.v) { - mlir::Location clauseLocation = converter.genLocation(clause.source); - if (const auto *collapseClause = - std::get_if(&clause.u)) { - const Fortran::parser::AccCollapseArg &arg = collapseClause->v; - const auto &force = std::get(arg.t); - if (force) - TODO(clauseLocation, "OpenACC collapse force modifier"); - const auto &intExpr = - std::get(arg.t); - const auto *expr = Fortran::semantics::GetExpr(intExpr); - const std::optional collapseValue = - Fortran::evaluate::ToInt64(*expr); - if (collapseValue) { - loopOp.setCollapseAttr(builder.getI64IntegerAttr(*collapseValue)); - } - } else if (std::get_if(&clause.u)) { - loopOp.setSeqAttr(builder.getUnitAttr()); - } else if (std::get_if( - &clause.u)) { - loopOp.setIndependentAttr(builder.getUnitAttr()); - } else if (std::get_if(&clause.u)) { - loopOp->setAttr(mlir::acc::LoopOp::getAutoAttrStrName(), - builder.getUnitAttr()); - } - } + if (!collapseValues.empty()) + loopOp.setCollapseAttr(builder.getI64ArrayAttr(collapseValues)); + if (!collapseDeviceTypes.empty()) + loopOp.setCollapseDeviceTypeAttr(builder.getArrayAttr(collapseDeviceTypes)); + return loopOp; } diff --git a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/flang/test/Lower/OpenACC/acc-kernels-loop.f90 index 93bc699031d5..b17f2e2c80b2 100644 --- a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-kernels-loop.f90 @@ -461,7 +461,7 @@ subroutine acc_kernels_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {seq} +! CHECK-NEXT: } attributes {seq = [#acc.device_type]} ! CHECK: acc.terminator ! CHECK-NEXT: }{{$}} @@ -474,7 +474,7 @@ subroutine acc_kernels_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {auto} +! CHECK-NEXT: } attributes {auto_ = [#acc.device_type]} ! CHECK: acc.terminator ! CHECK-NEXT: }{{$}} @@ -487,7 +487,7 @@ subroutine acc_kernels_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {independent} +! CHECK-NEXT: } attributes {independent = [#acc.device_type]} ! CHECK: acc.terminator ! CHECK-NEXT: }{{$}} @@ -497,10 +497,10 @@ subroutine acc_kernels_loop END DO ! CHECK: acc.kernels { -! CHECK: acc.loop gang { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {gang = [#acc.device_type]}{{$}} ! CHECK: acc.terminator ! CHECK-NEXT: }{{$}} @@ -511,7 +511,7 @@ subroutine acc_kernels_loop ! CHECK: acc.kernels { ! CHECK: [[GANGNUM1:%.*]] = arith.constant 8 : i32 -! CHECK-NEXT: acc.loop gang(num=[[GANGNUM1]] : i32) { +! CHECK-NEXT: acc.loop gang({num=[[GANGNUM1]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -525,7 +525,7 @@ subroutine acc_kernels_loop ! CHECK: acc.kernels { ! CHECK: [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref -! CHECK-NEXT: acc.loop gang(num=[[GANGNUM2]] : i32) { +! CHECK-NEXT: acc.loop gang({num=[[GANGNUM2]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -538,7 +538,7 @@ subroutine acc_kernels_loop END DO ! CHECK: acc.kernels { -! CHECK: acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) { +! CHECK: acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -550,10 +550,10 @@ subroutine acc_kernels_loop a(i) = b(i) END DO ! CHECK: acc.kernels { -! CHECK: acc.loop vector { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {vector = [#acc.device_type]}{{$}} ! CHECK: acc.terminator ! CHECK-NEXT: }{{$}} @@ -591,10 +591,10 @@ subroutine acc_kernels_loop END DO ! CHECK: acc.kernels { -! CHECK: acc.loop worker { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {worker = [#acc.device_type]}{{$}} ! CHECK: acc.terminator ! CHECK-NEXT: }{{$}} @@ -624,7 +624,7 @@ subroutine acc_kernels_loop ! CHECK: fir.do_loop ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {collapse = 2 : i64} +! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} ! CHECK: acc.terminator ! CHECK-NEXT: }{{$}} @@ -655,7 +655,7 @@ subroutine acc_kernels_loop ! CHECK: acc.kernels { ! CHECK: [[TILESIZE:%.*]] = arith.constant 2 : i32 -! CHECK: acc.loop tile([[TILESIZE]] : i32) { +! CHECK: acc.loop tile({[[TILESIZE]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -669,7 +669,7 @@ subroutine acc_kernels_loop ! CHECK: acc.kernels { ! CHECK: [[TILESIZEM1:%.*]] = arith.constant -1 : i32 -! CHECK: acc.loop tile([[TILESIZEM1]] : i32) { +! CHECK: acc.loop tile({[[TILESIZEM1]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -686,7 +686,7 @@ subroutine acc_kernels_loop ! CHECK: acc.kernels { ! CHECK: [[TILESIZE1:%.*]] = arith.constant 2 : i32 ! CHECK: [[TILESIZE2:%.*]] = arith.constant 2 : i32 -! CHECK: acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) { +! CHECK: acc.loop tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -699,7 +699,7 @@ subroutine acc_kernels_loop END DO ! CHECK: acc.kernels { -! CHECK: acc.loop tile(%{{.*}} : i32) { +! CHECK: acc.loop tile({%{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -714,7 +714,7 @@ subroutine acc_kernels_loop END DO ! CHECK: acc.kernels { -! CHECK: acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) { +! CHECK: acc.loop tile({%{{.*}} : i32, %{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} diff --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90 index 924574512da4..e7f65770498f 100644 --- a/flang/test/Lower/OpenACC/acc-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-loop.f90 @@ -1,6 +1,5 @@ ! This test checks lowering of OpenACC loop directive. -! RUN: bbc -fopenacc -emit-fir -hlfir=false %s -o - | FileCheck %s ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s ! CHECK-LABEL: acc.private.recipe @privatization_ref_10x10xf32 : !fir.ref> init { @@ -41,7 +40,7 @@ program acc_loop !CHECK: acc.loop { !CHECK: fir.do_loop !CHECK: acc.yield -!CHECK-NEXT: } attributes {seq} +!CHECK-NEXT: } attributes {seq = [#acc.device_type]} !$acc loop auto DO i = 1, n @@ -51,7 +50,7 @@ program acc_loop !CHECK: acc.loop { !CHECK: fir.do_loop !CHECK: acc.yield -!CHECK-NEXT: } attributes {auto} +!CHECK-NEXT: } attributes {auto_ = [#acc.device_type]} !$acc loop independent DO i = 1, n @@ -61,17 +60,17 @@ program acc_loop !CHECK: acc.loop { !CHECK: fir.do_loop !CHECK: acc.yield -!CHECK-NEXT: } attributes {independent} +!CHECK-NEXT: } attributes {independent = [#acc.device_type]} !$acc loop gang DO i = 1, n a(i) = b(i) END DO -!CHECK: acc.loop gang { +!CHECK: acc.loop { !CHECK: fir.do_loop !CHECK: acc.yield -!CHECK-NEXT: }{{$}} +!CHECK-NEXT: } attributes {gang = [#acc.device_type]}{{$}} !$acc loop gang(num: 8) DO i = 1, n @@ -79,7 +78,7 @@ program acc_loop END DO !CHECK: [[GANGNUM1:%.*]] = arith.constant 8 : i32 -!CHECK-NEXT: acc.loop gang(num=[[GANGNUM1]] : i32) { +!CHECK-NEXT: acc.loop gang({num=[[GANGNUM1]] : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -90,7 +89,7 @@ program acc_loop END DO !CHECK: [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref -!CHECK-NEXT: acc.loop gang(num=[[GANGNUM2]] : i32) { +!CHECK-NEXT: acc.loop gang({num=[[GANGNUM2]] : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -100,7 +99,7 @@ program acc_loop a(i) = b(i) END DO -!CHECK: acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) { +!CHECK: acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -110,10 +109,10 @@ program acc_loop a(i) = b(i) END DO -!CHECK: acc.loop vector { +!CHECK: acc.loop { !CHECK: fir.do_loop !CHECK: acc.yield -!CHECK-NEXT: }{{$}} +!CHECK-NEXT: } attributes {vector = [#acc.device_type]}{{$}} !$acc loop vector(128) DO i = 1, n @@ -142,10 +141,10 @@ program acc_loop a(i) = b(i) END DO -!CHECK: acc.loop worker { +!CHECK: acc.loop { !CHECK: fir.do_loop !CHECK: acc.yield -!CHECK-NEXT: }{{$}} +!CHECK-NEXT: } attributes {worker = [#acc.device_type]}{{$}} !$acc loop worker(128) DO i = 1, n @@ -193,7 +192,7 @@ program acc_loop a(i) = b(i) END DO !CHECK: [[TILESIZE:%.*]] = arith.constant 2 : i32 -!CHECK: acc.loop tile([[TILESIZE]] : i32) { +!CHECK: acc.loop tile({[[TILESIZE]] : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -203,7 +202,7 @@ program acc_loop a(i) = b(i) END DO !CHECK: [[TILESIZEM1:%.*]] = arith.constant -1 : i32 -!CHECK: acc.loop tile([[TILESIZEM1]] : i32) { +!CHECK: acc.loop tile({[[TILESIZEM1]] : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -217,7 +216,7 @@ program acc_loop !CHECK: [[TILESIZE1:%.*]] = arith.constant 2 : i32 !CHECK: [[TILESIZE2:%.*]] = arith.constant 2 : i32 -!CHECK: acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) { +!CHECK: acc.loop tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -227,7 +226,7 @@ program acc_loop a(i) = b(i) END DO -!CHECK: acc.loop tile(%{{.*}} : i32) { +!CHECK: acc.loop tile({%{{.*}} : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -239,7 +238,7 @@ program acc_loop END DO END DO -!CHECK: acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) { +!CHECK: acc.loop tile({%{{.*}} : i32, %{{.*}} : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -255,7 +254,7 @@ program acc_loop !CHECK: fir.do_loop !CHECK: fir.do_loop !CHECK: acc.yield -!CHECK-NEXT: } attributes {collapse = 2 : i64} +!CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} !$acc loop DO i = 1, n @@ -290,7 +289,7 @@ program acc_loop a(i) = b(i) END DO -!CHECK: acc.loop gang(dim=%{{.*}}, static=%{{.*}} : i32) { +!CHECK: acc.loop gang({dim=%{{.*}}, static=%{{.*}} : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} @@ -301,7 +300,7 @@ program acc_loop END DO !CHECK: [[GANGDIM1:%.*]] = arith.constant 1 : i32 -!CHECK-NEXT: acc.loop gang(dim=[[GANGDIM1]] : i32) { +!CHECK-NEXT: acc.loop gang({dim=[[GANGDIM1]] : i32}) { !CHECK: fir.do_loop !CHECK: acc.yield !CHECK-NEXT: }{{$}} diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 index deee7089033e..e9150a71f382 100644 --- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 @@ -476,7 +476,7 @@ subroutine acc_parallel_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {seq} +! CHECK-NEXT: } attributes {seq = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -489,7 +489,7 @@ subroutine acc_parallel_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {auto} +! CHECK-NEXT: } attributes {auto_ = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -502,7 +502,7 @@ subroutine acc_parallel_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {independent} +! CHECK-NEXT: } attributes {independent = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -512,10 +512,10 @@ subroutine acc_parallel_loop END DO ! CHECK: acc.parallel { -! CHECK: acc.loop gang { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {gang = [#acc.device_type]}{{$}} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -526,7 +526,7 @@ subroutine acc_parallel_loop ! CHECK: acc.parallel { ! CHECK: [[GANGNUM1:%.*]] = arith.constant 8 : i32 -! CHECK-NEXT: acc.loop gang(num=[[GANGNUM1]] : i32) { +! CHECK-NEXT: acc.loop gang({num=[[GANGNUM1]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -540,7 +540,7 @@ subroutine acc_parallel_loop ! CHECK: acc.parallel { ! CHECK: [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref -! CHECK-NEXT: acc.loop gang(num=[[GANGNUM2]] : i32) { +! CHECK-NEXT: acc.loop gang({num=[[GANGNUM2]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -553,7 +553,7 @@ subroutine acc_parallel_loop END DO ! CHECK: acc.parallel { -! CHECK: acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) { +! CHECK: acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -565,10 +565,10 @@ subroutine acc_parallel_loop a(i) = b(i) END DO ! CHECK: acc.parallel { -! CHECK: acc.loop vector { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {vector = [#acc.device_type]}{{$}} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -606,10 +606,10 @@ subroutine acc_parallel_loop END DO ! CHECK: acc.parallel { -! CHECK: acc.loop worker { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {worker = [#acc.device_type]}{{$}} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -639,7 +639,7 @@ subroutine acc_parallel_loop ! CHECK: fir.do_loop ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {collapse = 2 : i64} +! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -670,7 +670,7 @@ subroutine acc_parallel_loop ! CHECK: acc.parallel { ! CHECK: [[TILESIZE:%.*]] = arith.constant 2 : i32 -! CHECK: acc.loop tile([[TILESIZE]] : i32) { +! CHECK: acc.loop tile({[[TILESIZE]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -684,7 +684,7 @@ subroutine acc_parallel_loop ! CHECK: acc.parallel { ! CHECK: [[TILESIZEM1:%.*]] = arith.constant -1 : i32 -! CHECK: acc.loop tile([[TILESIZEM1]] : i32) { +! CHECK: acc.loop tile({[[TILESIZEM1]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -701,7 +701,7 @@ subroutine acc_parallel_loop ! CHECK: acc.parallel { ! CHECK: [[TILESIZE1:%.*]] = arith.constant 2 : i32 ! CHECK: [[TILESIZE2:%.*]] = arith.constant 2 : i32 -! CHECK: acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) { +! CHECK: acc.loop tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -714,7 +714,7 @@ subroutine acc_parallel_loop END DO ! CHECK: acc.parallel { -! CHECK: acc.loop tile(%{{.*}} : i32) { +! CHECK: acc.loop tile({%{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -729,7 +729,7 @@ subroutine acc_parallel_loop END DO ! CHECK: acc.parallel { -! CHECK: acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) { +! CHECK: acc.loop tile({%{{.*}} : i32, %{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90 index a8f7e1fa81ef..dcfa77c9f97d 100644 --- a/flang/test/Lower/OpenACC/acc-reduction.f90 +++ b/flang/test/Lower/OpenACC/acc-reduction.f90 @@ -743,7 +743,7 @@ end subroutine ! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref>) { -! CHECK: } attributes {collapse = 2 : i64} +! CHECK: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} subroutine acc_reduction_add_int_array_3d(a, b) integer :: a(100, 10, 2), b(100, 10, 2) @@ -765,7 +765,7 @@ end subroutine ! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 -> %[[RED_ARG1]] : !fir.ref>) -! CHECK: } attributes {collapse = 3 : i64} +! CHECK: } attributes {collapse = [3], collapseDeviceType = [#acc.device_type]} subroutine acc_reduction_add_float(a, b) real :: a(100), b @@ -938,7 +938,7 @@ end subroutine ! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%3, %5) -> !fir.ref> {name = "b"} ! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_min_section_ext100xext10_ref_100x10xf32 -> %[[RED_ARG1]] : !fir.ref>) -! CHECK: attributes {collapse = 2 : i64} +! CHECK: attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} subroutine acc_reduction_max_int(a, b) integer :: a(100) diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90 index 712bfc80ce38..6041e7fb1b49 100644 --- a/flang/test/Lower/OpenACC/acc-serial-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90 @@ -411,7 +411,7 @@ subroutine acc_serial_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {seq} +! CHECK-NEXT: } attributes {seq = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -424,7 +424,7 @@ subroutine acc_serial_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {auto} +! CHECK-NEXT: } attributes {auto_ = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -437,7 +437,7 @@ subroutine acc_serial_loop ! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {independent} +! CHECK-NEXT: } attributes {independent = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -447,10 +447,10 @@ subroutine acc_serial_loop END DO ! CHECK: acc.serial { -! CHECK: acc.loop gang { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {gang = [#acc.device_type]}{{$}} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -461,7 +461,7 @@ subroutine acc_serial_loop ! CHECK: acc.serial { ! CHECK: [[GANGNUM1:%.*]] = arith.constant 8 : i32 -! CHECK-NEXT: acc.loop gang(num=[[GANGNUM1]] : i32) { +! CHECK-NEXT: acc.loop gang({num=[[GANGNUM1]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -475,7 +475,7 @@ subroutine acc_serial_loop ! CHECK: acc.serial { ! CHECK: [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref -! CHECK-NEXT: acc.loop gang(num=[[GANGNUM2]] : i32) { +! CHECK-NEXT: acc.loop gang({num=[[GANGNUM2]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -488,7 +488,7 @@ subroutine acc_serial_loop END DO ! CHECK: acc.serial { -! CHECK: acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) { +! CHECK: acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -500,10 +500,10 @@ subroutine acc_serial_loop a(i) = b(i) END DO ! CHECK: acc.serial { -! CHECK: acc.loop vector { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {vector = [#acc.device_type]}{{$}} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -541,10 +541,10 @@ subroutine acc_serial_loop END DO ! CHECK: acc.serial { -! CHECK: acc.loop worker { +! CHECK: acc.loop { ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: }{{$}} +! CHECK-NEXT: } attributes {worker = [#acc.device_type]}{{$}} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -574,7 +574,7 @@ subroutine acc_serial_loop ! CHECK: fir.do_loop ! CHECK: fir.do_loop ! CHECK: acc.yield -! CHECK-NEXT: } attributes {collapse = 2 : i64} +! CHECK-NEXT: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -605,7 +605,7 @@ subroutine acc_serial_loop ! CHECK: acc.serial { ! CHECK: [[TILESIZE:%.*]] = arith.constant 2 : i32 -! CHECK: acc.loop tile([[TILESIZE]] : i32) { +! CHECK: acc.loop tile({[[TILESIZE]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -619,7 +619,7 @@ subroutine acc_serial_loop ! CHECK: acc.serial { ! CHECK: [[TILESIZEM1:%.*]] = arith.constant -1 : i32 -! CHECK: acc.loop tile([[TILESIZEM1]] : i32) { +! CHECK: acc.loop tile({[[TILESIZEM1]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -636,7 +636,7 @@ subroutine acc_serial_loop ! CHECK: acc.serial { ! CHECK: [[TILESIZE1:%.*]] = arith.constant 2 : i32 ! CHECK: [[TILESIZE2:%.*]] = arith.constant 2 : i32 -! CHECK: acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) { +! CHECK: acc.loop tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -649,7 +649,7 @@ subroutine acc_serial_loop END DO ! CHECK: acc.serial { -! CHECK: acc.loop tile(%{{.*}} : i32) { +! CHECK: acc.loop tile({%{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} @@ -664,7 +664,7 @@ subroutine acc_serial_loop END DO ! CHECK: acc.serial { -! CHECK: acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) { +! CHECK: acc.loop tile({%{{.*}} : i32, %{{.*}} : i32}) { ! CHECK: fir.do_loop ! CHECK: acc.yield ! CHECK-NEXT: }{{$}} diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index 1dd83e933034..e6954062a50e 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -196,6 +196,27 @@ def DeviceTypeArrayAttr : let constBuilderCall = ?; } +// Gang arg type enumeration +def OpenACC_GangArgNum : I32EnumAttrCase<"Num", 0, "Num">; +def OpenACC_GangArgDim : I32EnumAttrCase<"Dim", 1, "Dim">; +def OpenACC_GangArgStatic : I32EnumAttrCase<"Static", 2, "Static">; + +def OpenACC_GangArgType : I32EnumAttr<"GangArgType", + "Differentiate the different gang arg values", + [OpenACC_GangArgNum, OpenACC_GangArgDim, OpenACC_GangArgStatic]> { + let genSpecializedAttr = 0; + let cppNamespace = "::mlir::acc"; +} +def OpenACC_GangArgTypeAttr : EnumAttr { + let assemblyFormat = [{ ```<` $value `>` }]; +} +def GangArgTypeArrayAttr : + TypedArrayAttrBase { + let constBuilderCall = ?; +} + // Define a resource for the OpenACC runtime counters. def OpenACC_RuntimeCounters : Resource<"::mlir::acc::RuntimeCounters">; @@ -1462,7 +1483,7 @@ def OpenACC_LoopOp : OpenACC_Op<"loop", Example: ```mlir - acc.loop gang vector { + acc.loop { scf.for %arg3 = %c0 to %c10 step %c1 { scf.for %arg4 = %c0 to %c10 step %c1 { scf.for %arg5 = %c0 to %c10 step %c1 { @@ -1471,23 +1492,33 @@ def OpenACC_LoopOp : OpenACC_Op<"loop", } } acc.yield - } attributes { collapse = 3 } + } attributes { + collapse = [3], gang = [#acc.device_type], + vector = [#acc.device_type] + } ``` }]; - let arguments = (ins OptionalAttr:$collapse, - Optional:$gangNum, - Optional:$gangDim, - Optional:$gangStatic, - Optional:$workerNum, - Optional:$vectorLength, - UnitAttr:$seq, - UnitAttr:$independent, - UnitAttr:$auto_, - UnitAttr:$hasGang, - UnitAttr:$hasWorker, - UnitAttr:$hasVector, + let arguments = (ins + OptionalAttr:$collapse, + OptionalAttr:$collapseDeviceType, + Variadic:$gangOperands, + OptionalAttr:$gangOperandsArgType, + OptionalAttr:$gangOperandsSegments, + OptionalAttr:$gangOperandsDeviceType, + Variadic:$workerNumOperands, + OptionalAttr:$workerNumOperandsDeviceType, + Variadic:$vectorOperands, + OptionalAttr:$vectorOperandsDeviceType, + OptionalAttr:$seq, + OptionalAttr:$independent, + OptionalAttr:$auto_, + OptionalAttr:$gang, + OptionalAttr:$worker, + OptionalAttr:$vector, Variadic:$tileOperands, + OptionalAttr:$tileOperandsSegments, + OptionalAttr:$tileOperandsDeviceType, Variadic:$cacheOperands, Variadic:$privateOperands, OptionalAttr:$privatizations, @@ -1510,18 +1541,90 @@ def OpenACC_LoopOp : OpenACC_Op<"loop", /// The i-th data operand passed. Value getDataOperand(unsigned i); + + /// Return true if the op has the auto attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasAuto(); + /// Return true if the op has the auto attribute for the given device_type. + bool hasAuto(mlir::acc::DeviceType deviceType); + /// Return true if the op has the independent attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasIndependent(); + /// Return true if the op has the independent attribute for the given + /// device_type. + bool hasIndependent(mlir::acc::DeviceType deviceType); + /// Return true if the op has the seq attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasSeq(); + /// Return true if the op has the seq attribute for the given device_type. + bool hasSeq(mlir::acc::DeviceType deviceType); + + /// Return the value of the vector clause if present. + mlir::Value getVectorValue(); + /// Return the value of the vector clause for the given device_type + /// if present. + mlir::Value getVectorValue(mlir::acc::DeviceType deviceType); + /// Return true if the op has the vector attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasVector(); + /// Return true if the op has the vector attribute for the given + /// device_type. + bool hasVector(mlir::acc::DeviceType deviceType); + + /// Return the value of the worker clause if present. + mlir::Value getWorkerValue(); + /// Return the value of the worker clause for the given device_type + /// if present. + mlir::Value getWorkerValue(mlir::acc::DeviceType deviceType); + /// Return true if the op has the worker attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasWorker(); + /// Return true if the op has the worker attribute for the given + /// device_type. + bool hasWorker(mlir::acc::DeviceType deviceType); + + /// Return the values of the tile clause if present. + mlir::Operation::operand_range getTileValues(); + /// Return the values of the tile clause for the given device_type if + /// present. + mlir::Operation::operand_range + getTileValues(mlir::acc::DeviceType deviceType); + + /// Return the value of the collapse clause if present. + std::optional getCollapseValue(); + /// Return the value of the collapse clause for the given device_type + /// if present. + std::optional getCollapseValue(mlir::acc::DeviceType deviceType); + + /// Return true if the op has the gang attribute for the + /// mlir::acc::DeviceType::None device_type. + bool hasGang(); + /// Return true if the op has the gang attribute for the given + /// device_type. + bool hasGang(mlir::acc::DeviceType deviceType); + + /// Return the value of the worker clause if present. + mlir::Value getGangValue(mlir::acc::GangArgType gangArgType); + /// Return the value of the worker clause for the given device_type + /// if present. + mlir::Value getGangValue(mlir::acc::GangArgType gangArgType, mlir::acc::DeviceType deviceType); }]; let hasCustomAssemblyFormat = 1; let assemblyFormat = [{ oilist( - `gang` `` custom($gangNum, type($gangNum), $gangDim, type($gangDim), $gangStatic, type($gangStatic), $hasGang) - | `worker` `` custom($workerNum, type($workerNum), $hasWorker) - | `vector` `` custom($vectorLength, type($vectorLength), $hasVector) + `gang` `` `(` custom($gangOperands, type($gangOperands), + $gangOperandsArgType, $gangOperandsDeviceType, + $gangOperandsSegments) `)` + | `worker` `` `(` custom($workerNumOperands, + type($workerNumOperands), $workerNumOperandsDeviceType) `)` + | `vector` `` `(` custom($vectorOperands, + type($vectorOperands), $vectorOperandsDeviceType) `)` | `private` `(` custom( - $privateOperands, type($privateOperands), $privatizations) + $privateOperands, type($privateOperands), $privatizations) `)` + | `tile` `(` custom($tileOperands, + type($tileOperands), $tileOperandsDeviceType, $tileOperandsSegments) `)` - | `tile` `(` $tileOperands `:` type($tileOperands) `)` | `reduction` `(` custom( $reductionOperands, type($reductionOperands), $reductionRecipes) `)` diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp index 66605ead0529..c53673fa4260 100644 --- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp +++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp @@ -16,6 +16,7 @@ #include "mlir/IR/Matchers.h" #include "mlir/IR/OpImplementation.h" #include "mlir/Transforms/DialectConversion.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/TypeSwitch.h" using namespace mlir; @@ -886,6 +887,8 @@ static ParseResult parseDeviceTypeOperandsWithSegment( if (failed(parser.parseLBrace())) return failure(); + int32_t crtOperandsSize = operands.size(); + if (failed(parser.parseCommaSeparatedList( mlir::AsmParser::Delimiter::None, [&]() { if (parser.parseOperand(operands.emplace_back()) || @@ -895,7 +898,7 @@ static ParseResult parseDeviceTypeOperandsWithSegment( }))) return failure(); - seg.push_back(operands.size()); + seg.push_back(operands.size() - crtOperandsSize); if (failed(parser.parseRBrace())) return failure(); @@ -1207,16 +1210,19 @@ void acc::HostDataOp::getCanonicalizationPatterns(RewritePatternSet &results, // LoopOp //===----------------------------------------------------------------------===// -static ParseResult -parseGangValue(OpAsmParser &parser, llvm::StringRef keyword, - std::optional &value, - Type &valueType, bool &needComa, bool &newValue) { +static ParseResult parseGangValue( + OpAsmParser &parser, llvm::StringRef keyword, + llvm::SmallVectorImpl &operands, + llvm::SmallVectorImpl &types, + llvm::SmallVector &attributes, GangArgTypeAttr gangArgType, + bool &needComa, bool &newValue) { if (succeeded(parser.parseOptionalKeyword(keyword))) { if (parser.parseEqual()) return failure(); - value = OpAsmParser::UnresolvedOperand{}; - if (parser.parseOperand(*value) || parser.parseColonType(valueType)) + if (parser.parseOperand(operands.emplace_back()) || + parser.parseColonType(types.emplace_back())) return failure(); + attributes.push_back(gangArgType); needComa = true; newValue = true; } @@ -1224,19 +1230,27 @@ parseGangValue(OpAsmParser &parser, llvm::StringRef keyword, } static ParseResult parseGangClause( - OpAsmParser &parser, std::optional &gangNum, - Type &gangNumType, std::optional &gangDim, - Type &gangDimType, - std::optional &gangStatic, - Type &gangStaticType, UnitAttr &hasGang) { - hasGang = UnitAttr::get(parser.getBuilder().getContext()); - gangNum = std::nullopt; - gangDim = std::nullopt; - gangStatic = std::nullopt; + OpAsmParser &parser, + llvm::SmallVectorImpl &gangOperands, + llvm::SmallVectorImpl &gangOperandsType, mlir::ArrayAttr &gangArgType, + mlir::ArrayAttr &deviceType, mlir::DenseI32ArrayAttr &segments) { + llvm::SmallVector attributes; + llvm::SmallVector deviceTypeAttributes; + llvm::SmallVector seg; bool needComa = false; - // optional gang operands - if (succeeded(parser.parseOptionalLParen())) { + auto argNum = mlir::acc::GangArgTypeAttr::get(parser.getContext(), + mlir::acc::GangArgType::Num); + auto argDim = mlir::acc::GangArgTypeAttr::get(parser.getContext(), + mlir::acc::GangArgType::Dim); + auto argStatic = mlir::acc::GangArgTypeAttr::get( + parser.getContext(), mlir::acc::GangArgType::Static); + + do { + if (failed(parser.parseLBrace())) + return failure(); + + int32_t crtOperandsSize = gangOperands.size(); while (true) { bool newValue = false; bool needValue = false; @@ -1247,15 +1261,17 @@ static ParseResult parseGangClause( break; } - if (failed(parseGangValue(parser, LoopOp::getGangNumKeyword(), gangNum, - gangNumType, needComa, newValue))) + if (failed(parseGangValue(parser, LoopOp::getGangNumKeyword(), + gangOperands, gangOperandsType, attributes, + argNum, needComa, newValue))) return failure(); - if (failed(parseGangValue(parser, LoopOp::getGangDimKeyword(), gangDim, - gangDimType, needComa, newValue))) + if (failed(parseGangValue(parser, LoopOp::getGangDimKeyword(), + gangOperands, gangOperandsType, attributes, + argDim, needComa, newValue))) return failure(); if (failed(parseGangValue(parser, LoopOp::getGangStaticKeyword(), - gangStatic, gangStaticType, needComa, - newValue))) + gangOperands, gangOperandsType, attributes, + argStatic, needComa, newValue))) return failure(); if (!newValue && needValue) { @@ -1268,86 +1284,168 @@ static ParseResult parseGangClause( break; } - if (!gangNum && !gangDim && !gangStatic) { - parser.emitError(parser.getCurrentLocation(), - "expect at least one of num, dim or static values"); + if (gangOperands.empty()) + return parser.emitError( + parser.getCurrentLocation(), + "expect at least one of num, dim or static values"); + + if (failed(parser.parseRBrace())) return failure(); + + if (succeeded(parser.parseOptionalLSquare())) { + if (parser.parseAttribute(deviceTypeAttributes.emplace_back()) || + parser.parseRSquare()) + return failure(); + } else { + deviceTypeAttributes.push_back(mlir::acc::DeviceTypeAttr::get( + parser.getContext(), mlir::acc::DeviceType::None)); } - if (failed(parser.parseRParen())) - return failure(); - } + seg.push_back(gangOperands.size() - crtOperandsSize); + + } while (succeeded(parser.parseOptionalComma())); + + llvm::SmallVector arrayAttr(attributes.begin(), + attributes.end()); + gangArgType = ArrayAttr::get(parser.getContext(), arrayAttr); + + llvm::SmallVector deviceTypeAttr( + deviceTypeAttributes.begin(), deviceTypeAttributes.end()); + deviceType = ArrayAttr::get(parser.getContext(), deviceTypeAttr); + segments = DenseI32ArrayAttr::get(parser.getContext(), seg); return success(); } -void printGangClause(OpAsmPrinter &p, Operation *op, Value gangNum, - Type gangNumType, Value gangDim, Type gangDimType, - Value gangStatic, Type gangStaticType, UnitAttr hasGang) { - if (gangNum || gangStatic || gangDim) { - p << "("; - if (gangNum) { - p << LoopOp::getGangNumKeyword() << "=" << gangNum << " : " - << gangNumType; - if (gangStatic || gangDim) - p << ", "; - } - if (gangDim) { - p << LoopOp::getGangDimKeyword() << "=" << gangDim << " : " - << gangDimType; - if (gangStatic) +void printGangClause(OpAsmPrinter &p, Operation *op, + mlir::OperandRange operands, mlir::TypeRange types, + std::optional gangArgTypes, + std::optional deviceTypes, + std::optional segments) { + unsigned opIdx = 0; + for (unsigned i = 0; i < deviceTypes->size(); ++i) { + if (i != 0) + p << ", "; + p << "{"; + for (int32_t j = 0; j < (*segments)[i]; ++j) { + if (j != 0) p << ", "; + auto gangArgTypeAttr = + mlir::dyn_cast((*gangArgTypes)[opIdx]); + if (gangArgTypeAttr.getValue() == mlir::acc::GangArgType::Num) + p << LoopOp::getGangNumKeyword(); + else if (gangArgTypeAttr.getValue() == mlir::acc::GangArgType::Dim) + p << LoopOp::getGangDimKeyword(); + else if (gangArgTypeAttr.getValue() == mlir::acc::GangArgType::Static) + p << LoopOp::getGangStaticKeyword(); + p << "=" << operands[opIdx] << " : " << operands[opIdx].getType(); + ++opIdx; } - if (gangStatic) - p << LoopOp::getGangStaticKeyword() << "=" << gangStatic << " : " - << gangStaticType; - p << ")"; - } -} -static ParseResult -parseWorkerClause(OpAsmParser &parser, - std::optional &workerNum, - Type &workerNumType, UnitAttr &hasWorker) { - hasWorker = UnitAttr::get(parser.getBuilder().getContext()); - if (succeeded(parser.parseOptionalLParen())) { - workerNum = OpAsmParser::UnresolvedOperand{}; - if (parser.parseOperand(*workerNum) || - parser.parseColonType(workerNumType) || parser.parseRParen()) - return failure(); + p << "}"; + auto deviceTypeAttr = + mlir::dyn_cast((*deviceTypes)[i]); + if (deviceTypeAttr.getValue() != mlir::acc::DeviceType::None) + p << " [" << (*deviceTypes)[i] << "]"; } - return success(); } -void printWorkerClause(OpAsmPrinter &p, Operation *op, Value workerNum, - Type workerNumType, UnitAttr hasWorker) { - if (workerNum) - p << "(" << workerNum << " : " << workerNumType << ")"; +bool hasDuplicateDeviceTypes( + std::optional segments, + llvm::SmallSet &deviceTypes) { + if (!segments) + return false; + for (auto attr : *segments) { + auto deviceTypeAttr = mlir::dyn_cast(attr); + if (deviceTypes.contains(deviceTypeAttr.getValue())) + return true; + deviceTypes.insert(deviceTypeAttr.getValue()); + } + return false; } -static ParseResult -parseVectorClause(OpAsmParser &parser, - std::optional &vectorLength, - Type &vectorLengthType, UnitAttr &hasVector) { - hasVector = UnitAttr::get(parser.getBuilder().getContext()); - if (succeeded(parser.parseOptionalLParen())) { - vectorLength = OpAsmParser::UnresolvedOperand{}; - if (parser.parseOperand(*vectorLength) || - parser.parseColonType(vectorLengthType) || parser.parseRParen()) +/// Check for duplicates in the DeviceType array attribute. +LogicalResult checkDeviceTypes(mlir::ArrayAttr deviceTypes) { + llvm::SmallSet crtDeviceTypes; + if (!deviceTypes) + return success(); + for (auto attr : deviceTypes) { + auto deviceTypeAttr = + mlir::dyn_cast_or_null(attr); + if (!deviceTypeAttr) + return failure(); + if (crtDeviceTypes.contains(deviceTypeAttr.getValue())) return failure(); + crtDeviceTypes.insert(deviceTypeAttr.getValue()); } return success(); } -void printVectorClause(OpAsmPrinter &p, Operation *op, Value vectorLength, - Type vectorLengthType, UnitAttr hasVector) { - if (vectorLength) - p << "(" << vectorLength << " : " << vectorLengthType << ")"; -} - LogicalResult acc::LoopOp::verify() { + // Check collapse + if (getCollapseAttr() && !getCollapseDeviceTypeAttr()) + return emitOpError() << "collapse device_type attr must be define when" + << " collapse attr is present"; + + if (getCollapseAttr() && getCollapseDeviceTypeAttr() && + getCollapseAttr().getValue().size() != + getCollapseDeviceTypeAttr().getValue().size()) + return emitOpError() << "collapse attribute count must match collapse" + << " device_type count"; + if (failed(checkDeviceTypes(getCollapseDeviceTypeAttr()))) + return emitOpError() + << "duplicate device_type found in collapseDeviceType attribute"; + + // Check gang + if (!getGangOperands().empty()) { + if (!getGangOperandsArgType()) + return emitOpError() << "gangOperandsArgType attribute must be defined" + << " when gang operands are present"; + + if (getGangOperands().size() != + getGangOperandsArgTypeAttr().getValue().size()) + return emitOpError() << "gangOperandsArgType attribute count must match" + << " gangOperands count"; + } + if (getGangAttr() && failed(checkDeviceTypes(getGangAttr()))) + return emitOpError() << "duplicate device_type found in gang attribute"; + + if (failed(verifyDeviceTypeAndSegmentCountMatch( + *this, getGangOperands(), getGangOperandsSegmentsAttr(), + getGangOperandsDeviceTypeAttr(), "gang"))) + return failure(); + + // Check worker + if (failed(checkDeviceTypes(getWorkerAttr()))) + return emitOpError() << "duplicate device_type found in worker attribute"; + if (failed(checkDeviceTypes(getWorkerNumOperandsDeviceTypeAttr()))) + return emitOpError() << "duplicate device_type found in " + "workerNumOperandsDeviceType attribute"; + if (failed(verifyDeviceTypeCountMatch(*this, getWorkerNumOperands(), + getWorkerNumOperandsDeviceTypeAttr(), + "worker"))) + return failure(); + + // Check vector + if (failed(checkDeviceTypes(getVectorAttr()))) + return emitOpError() << "duplicate device_type found in vector attribute"; + if (failed(checkDeviceTypes(getVectorOperandsDeviceTypeAttr()))) + return emitOpError() << "duplicate device_type found in " + "vectorOperandsDeviceType attribute"; + if (failed(verifyDeviceTypeCountMatch(*this, getVectorOperands(), + getVectorOperandsDeviceTypeAttr(), + "vector"))) + return failure(); + + if (failed(verifyDeviceTypeAndSegmentCountMatch( + *this, getTileOperands(), getTileOperandsSegmentsAttr(), + getTileOperandsDeviceTypeAttr(), "tile"))) + return failure(); + // auto, independent and seq attribute are mutually exclusive. - if ((getAuto_() && (getIndependent() || getSeq())) || - (getIndependent() && getSeq())) { + llvm::SmallSet deviceTypes; + if (hasDuplicateDeviceTypes(getAuto_(), deviceTypes) || + hasDuplicateDeviceTypes(getIndependent(), deviceTypes) || + hasDuplicateDeviceTypes(getSeq(), deviceTypes)) { return emitError() << "only one of \"" << acc::LoopOp::getAutoAttrStrName() << "\", " << getIndependentAttrName() << ", " << getSeqAttrName() @@ -1355,8 +1453,24 @@ LogicalResult acc::LoopOp::verify() { } // Gang, worker and vector are incompatible with seq. - if (getSeq() && (getHasGang() || getHasWorker() || getHasVector())) - return emitError("gang, worker or vector cannot appear with the seq attr"); + if (getSeqAttr()) { + for (auto attr : getSeqAttr()) { + auto deviceTypeAttr = mlir::dyn_cast(attr); + if (hasVector(deviceTypeAttr.getValue()) || + getVectorValue(deviceTypeAttr.getValue()) || + hasWorker(deviceTypeAttr.getValue()) || + getWorkerValue(deviceTypeAttr.getValue()) || + hasGang(deviceTypeAttr.getValue()) || + getGangValue(mlir::acc::GangArgType::Num, + deviceTypeAttr.getValue()) || + getGangValue(mlir::acc::GangArgType::Dim, + deviceTypeAttr.getValue()) || + getGangValue(mlir::acc::GangArgType::Static, + deviceTypeAttr.getValue())) + return emitError() + << "gang, worker or vector cannot appear with the seq attr"; + } + } if (failed(checkSymOperandList( *this, getPrivatizations(), getPrivateOperands(), "private", @@ -1380,16 +1494,149 @@ unsigned LoopOp::getNumDataOperands() { } Value LoopOp::getDataOperand(unsigned i) { - unsigned numOptional = getGangNum() ? 1 : 0; - numOptional += getGangDim() ? 1 : 0; - numOptional += getGangStatic() ? 1 : 0; - numOptional += getVectorLength() ? 1 : 0; - numOptional += getWorkerNum() ? 1 : 0; + unsigned numOptional = getGangOperands().size(); + numOptional += getVectorOperands().size(); + numOptional += getWorkerNumOperands().size(); numOptional += getTileOperands().size(); numOptional += getCacheOperands().size(); return getOperand(numOptional + i); } +bool LoopOp::hasAuto() { return hasAuto(mlir::acc::DeviceType::None); } + +bool LoopOp::hasAuto(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getAuto_()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + +bool LoopOp::hasIndependent() { + return hasIndependent(mlir::acc::DeviceType::None); +} + +bool LoopOp::hasIndependent(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getIndependent()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + +bool LoopOp::hasSeq() { return hasSeq(mlir::acc::DeviceType::None); } + +bool LoopOp::hasSeq(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getSeq()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + +mlir::Value LoopOp::getVectorValue() { + return getVectorValue(mlir::acc::DeviceType::None); +} + +mlir::Value LoopOp::getVectorValue(mlir::acc::DeviceType deviceType) { + return getValueInDeviceTypeSegment(getVectorOperandsDeviceType(), + getVectorOperands(), deviceType); +} + +bool LoopOp::hasVector() { return hasVector(mlir::acc::DeviceType::None); } + +bool LoopOp::hasVector(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getVector()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + +mlir::Value LoopOp::getWorkerValue() { + return getWorkerValue(mlir::acc::DeviceType::None); +} + +mlir::Value LoopOp::getWorkerValue(mlir::acc::DeviceType deviceType) { + return getValueInDeviceTypeSegment(getWorkerNumOperandsDeviceType(), + getWorkerNumOperands(), deviceType); +} + +bool LoopOp::hasWorker() { return hasWorker(mlir::acc::DeviceType::None); } + +bool LoopOp::hasWorker(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getWorker()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + +mlir::Operation::operand_range LoopOp::getTileValues() { + return getTileValues(mlir::acc::DeviceType::None); +} + +mlir::Operation::operand_range +LoopOp::getTileValues(mlir::acc::DeviceType deviceType) { + return getValuesFromSegments(getTileOperandsDeviceType(), getTileOperands(), + getTileOperandsSegments(), deviceType); +} + +std::optional LoopOp::getCollapseValue() { + return getCollapseValue(mlir::acc::DeviceType::None); +} + +std::optional +LoopOp::getCollapseValue(mlir::acc::DeviceType deviceType) { + if (!getCollapseAttr()) + return std::nullopt; + if (auto pos = findSegment(getCollapseDeviceTypeAttr(), deviceType)) { + auto intAttr = + mlir::dyn_cast(getCollapseAttr().getValue()[*pos]); + return intAttr.getValue().getZExtValue(); + } + return std::nullopt; +} + +mlir::Value LoopOp::getGangValue(mlir::acc::GangArgType gangArgType) { + return getGangValue(gangArgType, mlir::acc::DeviceType::None); +} + +mlir::Value LoopOp::getGangValue(mlir::acc::GangArgType gangArgType, + mlir::acc::DeviceType deviceType) { + if (getGangOperands().empty()) + return {}; + if (auto pos = findSegment(*getGangOperandsDeviceType(), deviceType)) { + int32_t nbOperandsBefore = 0; + for (unsigned i = 0; i < *pos; ++i) + nbOperandsBefore += (*getGangOperandsSegments())[i]; + mlir::Operation::operand_range values = + getGangOperands() + .drop_front(nbOperandsBefore) + .take_front((*getGangOperandsSegments())[*pos]); + + int32_t argTypeIdx = nbOperandsBefore; + for (auto value : values) { + auto gangArgTypeAttr = mlir::dyn_cast( + (*getGangOperandsArgType())[argTypeIdx]); + if (gangArgTypeAttr.getValue() == gangArgType) + return value; + ++argTypeIdx; + } + } + return {}; +} + +bool LoopOp::hasGang() { return hasGang(mlir::acc::DeviceType::None); } + +bool LoopOp::hasGang(mlir::acc::DeviceType deviceType) { + if (auto arrayAttr = getGang()) { + if (findSegment(*arrayAttr, deviceType)) + return true; + } + return false; +} + //===----------------------------------------------------------------------===// // DataOp //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/OpenACC/invalid.mlir b/mlir/test/Dialect/OpenACC/invalid.mlir index c18d964b370f..5dcdb3a37e4e 100644 --- a/mlir/test/Dialect/OpenACC/invalid.mlir +++ b/mlir/test/Dialect/OpenACC/invalid.mlir @@ -1,58 +1,58 @@ // RUN: mlir-opt -split-input-file -verify-diagnostics %s // expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}} -acc.loop gang { +acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield -} attributes {seq} +} attributes {seq = [#acc.device_type], gang = [#acc.device_type]} // ----- // expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}} -acc.loop worker { +acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield -} attributes {seq} +} attributes {seq = [#acc.device_type], worker = [#acc.device_type]} // ----- // expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}} -acc.loop vector { +acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield -} attributes {seq} +} attributes {seq = [#acc.device_type], vector = [#acc.device_type]} // ----- // expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}} -acc.loop gang worker { +acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield -} attributes {seq} +} attributes {seq = [#acc.device_type], worker = [#acc.device_type], gang = [#acc.device_type]} // ----- // expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}} -acc.loop gang vector { +acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield -} attributes {seq} +} attributes {seq = [#acc.device_type], vector = [#acc.device_type], gang = [#acc.device_type]} // ----- // expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}} -acc.loop worker vector { +acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield -} attributes {seq} +} attributes {seq = [#acc.device_type], vector = [#acc.device_type], worker = [#acc.device_type]} // ----- // expected-error@+1 {{gang, worker or vector cannot appear with the seq attr}} -acc.loop gang worker vector { +acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield -} attributes {seq} +} attributes {seq = [#acc.device_type], vector = [#acc.device_type], worker = [#acc.device_type], gang = [#acc.device_type]} // ----- @@ -62,10 +62,31 @@ acc.loop { // ----- +// expected-error@+1 {{'acc.loop' op duplicate device_type found in gang attribute}} +acc.loop { + acc.yield +} attributes {gang = [#acc.device_type, #acc.device_type]} + +// ----- + +// expected-error@+1 {{'acc.loop' op duplicate device_type found in worker attribute}} +acc.loop { + acc.yield +} attributes {worker = [#acc.device_type, #acc.device_type]} + +// ----- + +// expected-error@+1 {{'acc.loop' op duplicate device_type found in vector attribute}} +acc.loop { + acc.yield +} attributes {vector = [#acc.device_type, #acc.device_type]} + +// ----- + // expected-error@+1 {{only one of "auto", "independent", "seq" can be present at the same time}} acc.loop { acc.yield -} attributes {auto_, seq} +} attributes {auto_ = [#acc.device_type], seq = [#acc.device_type]} // ----- @@ -368,7 +389,7 @@ acc.firstprivate.recipe @privatization_i32 : i32 init { // ----- // expected-error@+1 {{expected ')'}} -acc.loop gang(static=%i64Value: i64, num=%i64Value: i64 { +acc.loop gang({static=%i64Value: i64, num=%i64Value: i64} { "test.openacc_dummy_op"() : () -> () acc.yield } @@ -437,7 +458,7 @@ acc.reduction.recipe @reduction_i64 : i64 reduction_operator init { // ----- // expected-error@+1 {{new value expected after comma}} -acc.loop gang(static=%i64Value: i64, ) { +acc.loop gang({static=%i64Value: i64, ) { "test.openacc_dummy_op"() : () -> () acc.yield } @@ -454,7 +475,7 @@ func.func @fct1(%0 : !llvm.ptr) -> () { // ----- // expected-error@+1 {{expect at least one of num, dim or static values}} -acc.loop gang() { +acc.loop gang({}) { "test.openacc_dummy_op"() : () -> () acc.yield } diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir index 52375b1af314..ce5bfa490013 100644 --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -11,7 +11,7 @@ func.func @compute1(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x %async = arith.constant 1 : i64 acc.parallel async(%async: i64) { - acc.loop gang vector { + acc.loop { scf.for %arg3 = %c0 to %c10 step %c1 { scf.for %arg4 = %c0 to %c10 step %c1 { scf.for %arg5 = %c0 to %c10 step %c1 { @@ -25,7 +25,7 @@ func.func @compute1(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x } } acc.yield - } attributes { collapse = 3 } + } attributes { collapse = [3], collapseDeviceType = [#acc.device_type], vector = [#acc.device_type], gang = [#acc.device_type]} acc.yield } @@ -38,7 +38,7 @@ func.func @compute1(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x // CHECK-NEXT: %{{.*}} = arith.constant 1 : index // CHECK-NEXT: [[ASYNC:%.*]] = arith.constant 1 : i64 // CHECK-NEXT: acc.parallel async([[ASYNC]] : i64) { -// CHECK-NEXT: acc.loop gang vector { +// CHECK-NEXT: acc.loop { // CHECK-NEXT: scf.for %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} { // CHECK-NEXT: scf.for %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} { // CHECK-NEXT: scf.for %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} { @@ -52,7 +52,7 @@ func.func @compute1(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x // CHECK-NEXT: } // CHECK-NEXT: } // CHECK-NEXT: acc.yield -// CHECK-NEXT: } attributes {collapse = 3 : i64} +// CHECK-NEXT: } attributes {collapse = [3], collapseDeviceType = [#acc.device_type], gang = [#acc.device_type], vector = [#acc.device_type]} // CHECK-NEXT: acc.yield // CHECK-NEXT: } // CHECK-NEXT: return %{{.*}} : memref<10x10xf32> @@ -80,7 +80,7 @@ func.func @compute2(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x } } acc.yield - } attributes {seq} + } attributes {seq = [#acc.device_type]} acc.yield } @@ -106,7 +106,7 @@ func.func @compute2(%A: memref<10x10xf32>, %B: memref<10x10xf32>, %C: memref<10x // CHECK-NEXT: } // CHECK-NEXT: } // CHECK-NEXT: acc.yield -// CHECK-NEXT: } attributes {seq} +// CHECK-NEXT: } attributes {seq = [#acc.device_type]} // CHECK-NEXT: acc.yield // CHECK-NEXT: } // CHECK-NEXT: return %{{.*}} : memref<10x10xf32> @@ -138,9 +138,9 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x acc.data dataOperands(%pa, %pb, %pc, %pd: memref<10x10xf32>, memref<10x10xf32>, memref<10xf32>, memref<10xf32>) { %private = acc.private varPtr(%c : memref<10xf32>) -> memref<10xf32> acc.parallel num_gangs({%numGangs: i64}) num_workers(%numWorkers: i64 [#acc.device_type]) private(@privatization_memref_10_f32 -> %private : memref<10xf32>) { - acc.loop gang { + acc.loop { scf.for %x = %lb to %c10 step %st { - acc.loop worker { + acc.loop { scf.for %y = %lb to %c10 step %st { %axy = memref.load %a[%x, %y] : memref<10x10xf32> %bxy = memref.load %b[%x, %y] : memref<10x10xf32> @@ -148,7 +148,7 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x memref.store %tmp, %c[%y] : memref<10xf32> } acc.yield - } + } attributes {worker = [#acc.device_type]} acc.loop { // for i = 0 to 10 step 1 @@ -160,10 +160,10 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x memref.store %z, %d[%x] : memref<10xf32> } acc.yield - } attributes {seq} + } attributes {seq = [#acc.device_type]} } acc.yield - } + } attributes {gang = [#acc.device_type]} acc.yield } acc.terminator @@ -181,9 +181,9 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x // CHECK: acc.data dataOperands(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : memref<10x10xf32>, memref<10x10xf32>, memref<10xf32>, memref<10xf32>) { // CHECK-NEXT: %[[P_ARG2:.*]] = acc.private varPtr([[ARG2]] : memref<10xf32>) -> memref<10xf32> // CHECK-NEXT: acc.parallel num_gangs({[[NUMGANG]] : i64}) num_workers([[NUMWORKERS]] : i64 [#acc.device_type]) private(@privatization_memref_10_f32 -> %[[P_ARG2]] : memref<10xf32>) { -// CHECK-NEXT: acc.loop gang { +// CHECK-NEXT: acc.loop { // CHECK-NEXT: scf.for %{{.*}} = [[C0]] to [[C10]] step [[C1]] { -// CHECK-NEXT: acc.loop worker { +// CHECK-NEXT: acc.loop { // CHECK-NEXT: scf.for %{{.*}} = [[C0]] to [[C10]] step [[C1]] { // CHECK-NEXT: %{{.*}} = memref.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<10x10xf32> // CHECK-NEXT: %{{.*}} = memref.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<10x10xf32> @@ -191,7 +191,7 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x // CHECK-NEXT: memref.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32> // CHECK-NEXT: } // CHECK-NEXT: acc.yield -// CHECK-NEXT: } +// CHECK-NEXT: } attributes {worker = [#acc.device_type]} // CHECK-NEXT: acc.loop { // CHECK-NEXT: scf.for %{{.*}} = [[C0]] to [[C10]] step [[C1]] { // CHECK-NEXT: %{{.*}} = memref.load %{{.*}}[%{{.*}}] : memref<10xf32> @@ -200,10 +200,10 @@ func.func @compute3(%a: memref<10x10xf32>, %b: memref<10x10xf32>, %c: memref<10x // CHECK-NEXT: memref.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32> // CHECK-NEXT: } // CHECK-NEXT: acc.yield -// CHECK-NEXT: } attributes {seq} +// CHECK-NEXT: } attributes {seq = [#acc.device_type]} // CHECK-NEXT: } // CHECK-NEXT: acc.yield -// CHECK-NEXT: } +// CHECK-NEXT: } attributes {gang = [#acc.device_type]} // CHECK-NEXT: acc.yield // CHECK-NEXT: } // CHECK-NEXT: acc.terminator @@ -218,15 +218,15 @@ func.func @testloopop(%a : memref<10xf32>) -> () { %i32Value = arith.constant 128 : i32 %idxValue = arith.constant 8 : index - acc.loop gang worker vector { + acc.loop { "test.openacc_dummy_op"() : () -> () acc.yield - } - acc.loop gang(num=%i64Value: i64) { + } attributes {vector = [#acc.device_type], worker = [#acc.device_type], gang = [#acc.device_type]} + acc.loop gang({num=%i64Value: i64}) { "test.openacc_dummy_op"() : () -> () acc.yield } - acc.loop gang(static=%i64Value: i64) { + acc.loop gang({static=%i64Value: i64}) { "test.openacc_dummy_op"() : () -> () acc.yield } @@ -254,31 +254,31 @@ func.func @testloopop(%a : memref<10xf32>) -> () { "test.openacc_dummy_op"() : () -> () acc.yield } - acc.loop gang(num=%i64Value: i64) worker vector { + acc.loop gang({num=%i64Value: i64}) { "test.openacc_dummy_op"() : () -> () acc.yield - } - acc.loop gang(num=%i64Value: i64, static=%i64Value: i64) worker(%i64Value: i64) vector(%i64Value: i64) { + } attributes {vector = [#acc.device_type], worker = [#acc.device_type]} + acc.loop gang({num=%i64Value: i64, static=%i64Value: i64}) worker(%i64Value: i64) vector(%i64Value: i64) { "test.openacc_dummy_op"() : () -> () acc.yield } - acc.loop gang(num=%i32Value: i32, static=%idxValue: index) { + acc.loop gang({num=%i32Value: i32, static=%idxValue: index}) { "test.openacc_dummy_op"() : () -> () acc.yield } - acc.loop tile(%i64Value, %i64Value : i64, i64) { + acc.loop tile({%i64Value : i64, %i64Value : i64}) { "test.openacc_dummy_op"() : () -> () acc.yield } - acc.loop tile(%i32Value, %i32Value : i32, i32) { + acc.loop tile({%i32Value : i32, %i32Value : i32}) { "test.openacc_dummy_op"() : () -> () acc.yield } - acc.loop gang(static=%i64Value: i64, num=%i64Value: i64) { + acc.loop gang({static=%i64Value: i64, num=%i64Value: i64}) { "test.openacc_dummy_op"() : () -> () acc.yield } - acc.loop gang(dim=%i64Value : i64, static=%i64Value: i64) { + acc.loop gang({dim=%i64Value : i64, static=%i64Value: i64}) { "test.openacc_dummy_op"() : () -> () acc.yield } @@ -293,15 +293,15 @@ func.func @testloopop(%a : memref<10xf32>) -> () { // CHECK: [[I64VALUE:%.*]] = arith.constant 1 : i64 // CHECK-NEXT: [[I32VALUE:%.*]] = arith.constant 128 : i32 // CHECK-NEXT: [[IDXVALUE:%.*]] = arith.constant 8 : index -// CHECK: acc.loop gang worker vector { +// CHECK: acc.loop { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield -// CHECK-NEXT: } -// CHECK: acc.loop gang(num=[[I64VALUE]] : i64) { +// CHECK-NEXT: } attributes {gang = [#acc.device_type], vector = [#acc.device_type], worker = [#acc.device_type]} +// CHECK: acc.loop gang({num=[[I64VALUE]] : i64}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -// CHECK: acc.loop gang(static=[[I64VALUE]] : i64) { +// CHECK: acc.loop gang({static=[[I64VALUE]] : i64}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } @@ -329,31 +329,31 @@ func.func @testloopop(%a : memref<10xf32>) -> () { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -// CHECK: acc.loop gang(num=[[I64VALUE]] : i64) worker vector { +// CHECK: acc.loop gang({num=[[I64VALUE]] : i64}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield -// CHECK-NEXT: } -// CHECK: acc.loop gang(num=[[I64VALUE]] : i64, static=[[I64VALUE]] : i64) worker([[I64VALUE]] : i64) vector([[I64VALUE]] : i64) { +// CHECK-NEXT: } attributes {vector = [#acc.device_type], worker = [#acc.device_type]} +// CHECK: acc.loop gang({num=[[I64VALUE]] : i64, static=[[I64VALUE]] : i64}) worker([[I64VALUE]] : i64) vector([[I64VALUE]] : i64) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -// CHECK: acc.loop gang(num=[[I32VALUE]] : i32, static=[[IDXVALUE]] : index) { +// CHECK: acc.loop gang({num=[[I32VALUE]] : i32, static=[[IDXVALUE]] : index}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -// CHECK: acc.loop tile([[I64VALUE]], [[I64VALUE]] : i64, i64) { +// CHECK: acc.loop tile({[[I64VALUE]] : i64, [[I64VALUE]] : i64}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -// CHECK: acc.loop tile([[I32VALUE]], [[I32VALUE]] : i32, i32) { +// CHECK: acc.loop tile({[[I32VALUE]] : i32, [[I32VALUE]] : i32}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -// CHECK: acc.loop gang(num=[[I64VALUE]] : i64, static=[[I64VALUE]] : i64) { +// CHECK: acc.loop gang({static=[[I64VALUE]] : i64, num=[[I64VALUE]] : i64}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -// CHECK: acc.loop gang(dim=[[I64VALUE]] : i64, static=[[I64VALUE]] : i64) { +// CHECK: acc.loop gang({dim=[[I64VALUE]] : i64, static=[[I64VALUE]] : i64}) { // CHECK-NEXT: "test.openacc_dummy_op"() : () -> () // CHECK-NEXT: acc.yield // CHECK-NEXT: } -- GitLab From 5fd18bdef9e1f18d6069a542551f046f1a179b38 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 4 Jan 2024 16:47:09 -0800 Subject: [PATCH 057/728] Revert "XFAIL test with dsymutil" This reverts commit c041fa1093c3ad7be040fb362a10ca3900c698a4 as Adrian added support to dsymutil. --- .../functionalities/inline-sourcefile/TestInlineSourceFiles.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py b/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py index ce7ac6fc503e..20ed0ce00661 100644 --- a/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py +++ b/lldb/test/API/functionalities/inline-sourcefile/TestInlineSourceFiles.py @@ -8,8 +8,6 @@ from lldbsuite.test import lldbutil class InlineSourceFilesTestCase(TestBase): @skipIf(compiler="gcc") @skipIf(compiler="clang", compiler_version=["<", "18.0"]) - # dsymutil doesn't yet copy the sources - @expectedFailureDarwin(debug_info=["dsym"]) def test(self): """Test DWARF inline source files.""" self.build() -- GitLab From c1eef483b2c1ab2564e0ee1e4d1a30db11f8049f Mon Sep 17 00:00:00 2001 From: Uday Bondhugula Date: Fri, 5 Jan 2024 06:35:22 +0530 Subject: [PATCH 058/728] [MLIR] Support interrupting AffineExpr walks (#74792) Support WalkResult for AffineExpr walk and support interrupting walks along the lines of Operation::walk. This allows interrupted walks when a condition is met. Also, switch from std::function to llvm::function_ref for the walk function. --- mlir/include/mlir/IR/AffineExpr.h | 19 +++++++- mlir/include/mlir/IR/AffineExprVisitor.h | 56 ++++++++++++++++++---- mlir/lib/Dialect/Affine/Utils/Utils.cpp | 60 ++++++++++++------------ mlir/lib/IR/AffineExpr.cpp | 41 +++++++++++----- mlir/test/IR/affine-walk.mlir | 9 ++++ mlir/test/lib/IR/CMakeLists.txt | 1 + mlir/test/lib/IR/TestAffineWalk.cpp | 57 ++++++++++++++++++++++ mlir/tools/mlir-opt/mlir-opt.cpp | 12 +++-- 8 files changed, 196 insertions(+), 59 deletions(-) create mode 100644 mlir/test/IR/affine-walk.mlir create mode 100644 mlir/test/lib/IR/TestAffineWalk.cpp diff --git a/mlir/include/mlir/IR/AffineExpr.h b/mlir/include/mlir/IR/AffineExpr.h index 40e9d28ce5d3..63314cc75635 100644 --- a/mlir/include/mlir/IR/AffineExpr.h +++ b/mlir/include/mlir/IR/AffineExpr.h @@ -14,6 +14,7 @@ #ifndef MLIR_IR_AFFINEEXPR_H #define MLIR_IR_AFFINEEXPR_H +#include "mlir/IR/Visitors.h" #include "mlir/Support/LLVM.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/Hashing.h" @@ -123,8 +124,13 @@ public: /// Return true if the affine expression involves AffineSymbolExpr `position`. bool isFunctionOfSymbol(unsigned position) const; - /// Walk all of the AffineExpr's in this expression in postorder. - void walk(std::function callback) const; + /// Walk all of the AffineExpr's in this expression in postorder. This allows + /// a lambda walk function that can either return `void` or a WalkResult. With + /// a WalkResult, interrupting is supported. + template > + RetT walk(FnT &&callback) const { + return walk(*this, callback); + } /// This method substitutes any uses of dimensions and symbols (e.g. /// dim#0 with dimReplacements[0]) and returns the modified expression tree. @@ -202,6 +208,15 @@ public: protected: ImplType *expr{nullptr}; + +private: + /// A trampoline for the templated non-static AffineExpr::walk method to + /// dispatch lambda `callback`'s of either a void result type or a + /// WalkResult type. Walk all of the AffineExprs in `e` in postorder. Users + /// should use the regular (non-static) `walk` method. + template + static WalkRetTy walk(AffineExpr e, + function_ref callback); }; /// Affine binary operation expression. An affine binary operation could be an diff --git a/mlir/include/mlir/IR/AffineExprVisitor.h b/mlir/include/mlir/IR/AffineExprVisitor.h index 2860e73c8f42..3e1bbb4b3fa0 100644 --- a/mlir/include/mlir/IR/AffineExprVisitor.h +++ b/mlir/include/mlir/IR/AffineExprVisitor.h @@ -30,6 +30,9 @@ namespace mlir { /// functions in your class. This class is defined in terms of statically /// resolved overloading, not virtual functions. /// +/// The visitor is templated on its return type (`RetTy`). With a WalkResult +/// return type, the visitor supports interrupting walks. +/// /// For example, here is a visitor that counts the number of for AffineDimExprs /// in an AffineExpr. /// @@ -65,7 +68,6 @@ namespace mlir { /// virtual function call overhead. Defining and using a AffineExprVisitor is /// just as efficient as having your own switch instruction over the instruction /// opcode. - template class AffineExprVisitorBase { public: @@ -136,6 +138,8 @@ public: RetTy visitSymbolExpr(AffineSymbolExpr expr) { return RetTy(); } }; +/// See documentation for AffineExprVisitorBase. This visitor supports +/// interrupting walks when a `WalkResult` is used for `RetTy`. template class AffineExprVisitor : public AffineExprVisitorBase { //===--------------------------------------------------------------------===// @@ -150,27 +154,52 @@ public: switch (expr.getKind()) { case AffineExprKind::Add: { auto binOpExpr = cast(expr); - walkOperandsPostOrder(binOpExpr); + if constexpr (std::is_same::value) { + if (walkOperandsPostOrder(binOpExpr).wasInterrupted()) + return WalkResult::interrupt(); + } else { + walkOperandsPostOrder(binOpExpr); + } return self->visitAddExpr(binOpExpr); } case AffineExprKind::Mul: { auto binOpExpr = cast(expr); - walkOperandsPostOrder(binOpExpr); + if constexpr (std::is_same::value) { + if (walkOperandsPostOrder(binOpExpr).wasInterrupted()) + return WalkResult::interrupt(); + } else { + walkOperandsPostOrder(binOpExpr); + } return self->visitMulExpr(binOpExpr); } case AffineExprKind::Mod: { auto binOpExpr = cast(expr); - walkOperandsPostOrder(binOpExpr); + if constexpr (std::is_same::value) { + if (walkOperandsPostOrder(binOpExpr).wasInterrupted()) + return WalkResult::interrupt(); + } else { + walkOperandsPostOrder(binOpExpr); + } return self->visitModExpr(binOpExpr); } case AffineExprKind::FloorDiv: { auto binOpExpr = cast(expr); - walkOperandsPostOrder(binOpExpr); + if constexpr (std::is_same::value) { + if (walkOperandsPostOrder(binOpExpr).wasInterrupted()) + return WalkResult::interrupt(); + } else { + walkOperandsPostOrder(binOpExpr); + } return self->visitFloorDivExpr(binOpExpr); } case AffineExprKind::CeilDiv: { auto binOpExpr = cast(expr); - walkOperandsPostOrder(binOpExpr); + if constexpr (std::is_same::value) { + if (walkOperandsPostOrder(binOpExpr).wasInterrupted()) + return WalkResult::interrupt(); + } else { + walkOperandsPostOrder(binOpExpr); + } return self->visitCeilDivExpr(binOpExpr); } case AffineExprKind::Constant: @@ -186,8 +215,19 @@ public: private: // Walk the operands - each operand is itself walked in post order. RetTy walkOperandsPostOrder(AffineBinaryOpExpr expr) { - walkPostOrder(expr.getLHS()); - walkPostOrder(expr.getRHS()); + if constexpr (std::is_same::value) { + if (walkPostOrder(expr.getLHS()).wasInterrupted()) + return WalkResult::interrupt(); + } else { + walkPostOrder(expr.getLHS()); + } + if constexpr (std::is_same::value) { + if (walkPostOrder(expr.getLHS()).wasInterrupted()) + return WalkResult::interrupt(); + return WalkResult::advance(); + } else { + return walkPostOrder(expr.getRHS()); + } } }; diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp index 50a052fb8b74..578d03c62928 100644 --- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp +++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp @@ -1561,22 +1561,21 @@ static LogicalResult getTileSizePos( /// memref<4x?xf32, #map0> ==> memref<4x?x?xf32> static bool isNormalizedMemRefDynamicDim(unsigned dim, AffineMap layoutMap, - SmallVectorImpl &inMemrefTypeDynDims, - MLIRContext *context) { - bool isDynamicDim = false; + SmallVectorImpl &inMemrefTypeDynDims) { AffineExpr expr = layoutMap.getResults()[dim]; // Check if affine expr of the dimension includes dynamic dimension of input // memrefType. - expr.walk([&inMemrefTypeDynDims, &isDynamicDim, &context](AffineExpr e) { - if (isa(e)) { - for (unsigned dm : inMemrefTypeDynDims) { - if (e == getAffineDimExpr(dm, context)) { - isDynamicDim = true; - } - } - } - }); - return isDynamicDim; + MLIRContext *context = layoutMap.getContext(); + return expr + .walk([&](AffineExpr e) { + if (isa(e) && + llvm::any_of(inMemrefTypeDynDims, [&](unsigned dim) { + return e == getAffineDimExpr(dim, context); + })) + return WalkResult::interrupt(); + return WalkResult::advance(); + }) + .wasInterrupted(); } /// Create affine expr to calculate dimension size for a tiled-layout map. @@ -1792,29 +1791,28 @@ MemRefType mlir::affine::normalizeMemRefType(MemRefType memrefType) { MLIRContext *context = memrefType.getContext(); for (unsigned d = 0; d < newRank; ++d) { // Check if this dimension is dynamic. - bool isDynDim = - isNormalizedMemRefDynamicDim(d, layoutMap, memrefTypeDynDims, context); - if (isDynDim) { + if (bool isDynDim = + isNormalizedMemRefDynamicDim(d, layoutMap, memrefTypeDynDims)) { newShape[d] = ShapedType::kDynamic; - } else { - // The lower bound for the shape is always zero. - std::optional ubConst = fac.getConstantBound64(BoundType::UB, d); - // For a static memref and an affine map with no symbols, this is - // always bounded. However, when we have symbols, we may not be able to - // obtain a constant upper bound. Also, mapping to a negative space is - // invalid for normalization. - if (!ubConst.has_value() || *ubConst < 0) { - LLVM_DEBUG(llvm::dbgs() - << "can't normalize map due to unknown/invalid upper bound"); - return memrefType; - } - // If dimension of new memrefType is dynamic, the value is -1. - newShape[d] = *ubConst + 1; + continue; + } + // The lower bound for the shape is always zero. + std::optional ubConst = fac.getConstantBound64(BoundType::UB, d); + // For a static memref and an affine map with no symbols, this is + // always bounded. However, when we have symbols, we may not be able to + // obtain a constant upper bound. Also, mapping to a negative space is + // invalid for normalization. + if (!ubConst.has_value() || *ubConst < 0) { + LLVM_DEBUG(llvm::dbgs() + << "can't normalize map due to unknown/invalid upper bound"); + return memrefType; } + // If dimension of new memrefType is dynamic, the value is -1. + newShape[d] = *ubConst + 1; } // Create the new memref type after trivializing the old layout map. - MemRefType newMemRefType = + auto newMemRefType = MemRefType::Builder(memrefType) .setShape(newShape) .setLayout(AffineMapAttr::get( diff --git a/mlir/lib/IR/AffineExpr.cpp b/mlir/lib/IR/AffineExpr.cpp index 038ceea286a3..a90b264a8edd 100644 --- a/mlir/lib/IR/AffineExpr.cpp +++ b/mlir/lib/IR/AffineExpr.cpp @@ -26,22 +26,37 @@ MLIRContext *AffineExpr::getContext() const { return expr->context; } AffineExprKind AffineExpr::getKind() const { return expr->kind; } -/// Walk all of the AffineExprs in this subgraph in postorder. -void AffineExpr::walk(std::function callback) const { - struct AffineExprWalker : public AffineExprVisitor { - std::function callback; - - AffineExprWalker(std::function callback) - : callback(std::move(callback)) {} - - void visitAffineBinaryOpExpr(AffineBinaryOpExpr expr) { callback(expr); } - void visitConstantExpr(AffineConstantExpr expr) { callback(expr); } - void visitDimExpr(AffineDimExpr expr) { callback(expr); } - void visitSymbolExpr(AffineSymbolExpr expr) { callback(expr); } +/// Walk all of the AffineExprs in `e` in postorder. This is a private factory +/// method to help handle lambda walk functions. Users should use the regular +/// (non-static) `walk` method. +template +WalkRetTy mlir::AffineExpr::walk(AffineExpr e, + function_ref callback) { + struct AffineExprWalker + : public AffineExprVisitor { + function_ref callback; + + AffineExprWalker(function_ref callback) + : callback(callback) {} + + WalkRetTy visitAffineBinaryOpExpr(AffineBinaryOpExpr expr) { + return callback(expr); + } + WalkRetTy visitConstantExpr(AffineConstantExpr expr) { + return callback(expr); + } + WalkRetTy visitDimExpr(AffineDimExpr expr) { return callback(expr); } + WalkRetTy visitSymbolExpr(AffineSymbolExpr expr) { return callback(expr); } }; - AffineExprWalker(std::move(callback)).walkPostOrder(*this); + return AffineExprWalker(callback).walkPostOrder(e); } +// Explicitly instantiate for the two supported return types. +template void mlir::AffineExpr::walk(AffineExpr e, + function_ref callback); +template WalkResult +mlir::AffineExpr::walk(AffineExpr e, + function_ref callback); // Dispatch affine expression construction based on kind. AffineExpr mlir::getAffineBinaryOpExpr(AffineExprKind kind, AffineExpr lhs, diff --git a/mlir/test/IR/affine-walk.mlir b/mlir/test/IR/affine-walk.mlir new file mode 100644 index 000000000000..1de675ac70be --- /dev/null +++ b/mlir/test/IR/affine-walk.mlir @@ -0,0 +1,9 @@ +// RUN: mlir-opt -test-affine-walk -verify-diagnostics %s + +// Test affine walk interrupt. A remark should be printed only for the first mod +// expression encountered in post order. + +#map = affine_map<(i, j) -> ((i mod 4) mod 2, j)> + +"test.check_first_mod"() {"map" = #map} : () -> () +// expected-remark@-1 {{mod expression}} diff --git a/mlir/test/lib/IR/CMakeLists.txt b/mlir/test/lib/IR/CMakeLists.txt index 69c63fd7e524..faaa3bb8db24 100644 --- a/mlir/test/lib/IR/CMakeLists.txt +++ b/mlir/test/lib/IR/CMakeLists.txt @@ -1,5 +1,6 @@ # Exclude tests from libMLIR.so add_mlir_library(MLIRTestIR + TestAffineWalk.cpp TestBytecodeRoundtrip.cpp TestBuiltinAttributeInterfaces.cpp TestBuiltinDistinctAttributes.cpp diff --git a/mlir/test/lib/IR/TestAffineWalk.cpp b/mlir/test/lib/IR/TestAffineWalk.cpp new file mode 100644 index 000000000000..8361b48ce428 --- /dev/null +++ b/mlir/test/lib/IR/TestAffineWalk.cpp @@ -0,0 +1,57 @@ +//===- TestAffineWalk.cpp - Pass to test affine walks +//----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Pass/Pass.h" + +#include "mlir/IR/BuiltinOps.h" + +using namespace mlir; + +namespace { +/// A test pass for verifying walk interrupts. +struct TestAffineWalk + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestAffineWalk) + + void runOnOperation() override; + StringRef getArgument() const final { return "test-affine-walk"; } + StringRef getDescription() const final { return "Test affine walk method."; } +}; +} // namespace + +/// Emits a remark for the first `map`'s result expression that contains a +/// mod expression. +static void checkMod(AffineMap map, Location loc) { + for (AffineExpr e : map.getResults()) { + e.walk([&](AffineExpr s) { + if (s.getKind() == mlir::AffineExprKind::Mod) { + emitRemark(loc, "mod expression: "); + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + } +} + +void TestAffineWalk::runOnOperation() { + auto m = getOperation(); + // Test whether the walk is being correctly interrupted. + m.walk([](Operation *op) { + for (NamedAttribute attr : op->getAttrs()) { + auto mapAttr = attr.getValue().dyn_cast(); + if (!mapAttr) + return; + checkMod(mapAttr.getAffineMap(), op->getLoc()); + } + }); +} + +namespace mlir { +void registerTestAffineWalk() { PassRegistration(); } +} // namespace mlir diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp index 3fce7a7eea9e..09ff66e07957 100644 --- a/mlir/tools/mlir-opt/mlir-opt.cpp +++ b/mlir/tools/mlir-opt/mlir-opt.cpp @@ -44,11 +44,12 @@ void registerSymbolTestPasses(); void registerRegionTestPasses(); void registerTestAffineDataCopyPass(); void registerTestAffineReifyValueBoundsPass(); +void registerTestAffineLoopUnswitchingPass(); +void registerTestAffineWalk(); void registerTestBytecodeRoundtripPasses(); void registerTestDecomposeAffineOpPass(); -void registerTestAffineLoopUnswitchingPass(); -void registerTestGpuLoweringPasses(); void registerTestFunc(); +void registerTestGpuLoweringPasses(); void registerTestGpuMemoryPromotionPass(); void registerTestLoopPermutationPass(); void registerTestMatchers(); @@ -167,12 +168,13 @@ void registerTestPasses() { registerSymbolTestPasses(); registerRegionTestPasses(); registerTestAffineDataCopyPass(); - registerTestAffineReifyValueBoundsPass(); - registerTestDecomposeAffineOpPass(); registerTestAffineLoopUnswitchingPass(); - registerTestGpuLoweringPasses(); + registerTestAffineReifyValueBoundsPass(); + registerTestAffineWalk(); registerTestBytecodeRoundtripPasses(); + registerTestDecomposeAffineOpPass(); registerTestFunc(); + registerTestGpuLoweringPasses(); registerTestGpuMemoryPromotionPass(); registerTestLoopPermutationPass(); registerTestMatchers(); -- GitLab From 47685633a7dc74451acbc551b111929166d4d0bd Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Fri, 5 Jan 2024 08:35:07 +0700 Subject: [PATCH 059/728] AMDGPU: Make v4bf16 a legal type (#76217) Gets a few code quality improvements. A few cases are worse from losing load narrowing. Depends #76213 #76214 #76215 --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 19 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 25 +- llvm/lib/Target/AMDGPU/SIInstructions.td | 13 + llvm/test/CodeGen/AMDGPU/bf16.ll | 11487 +++++++--------- llvm/test/CodeGen/AMDGPU/function-args.ll | 61 +- .../CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll | 101 +- llvm/test/CodeGen/AMDGPU/select-undef.ll | 3 +- .../CodeGen/AMDGPU/vector_shuffle.packed.ll | 399 +- 8 files changed, 5678 insertions(+), 6430 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index b70d33d58b74..2f663571a8f9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -389,15 +389,16 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, Custom); setOperationAction( ISD::EXTRACT_SUBVECTOR, - {MVT::v2f16, MVT::v2i16, MVT::v4f16, MVT::v4i16, MVT::v2f32, - MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32, MVT::v4i32, - MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32, MVT::v7f32, - MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32, MVT::v9i32, - MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, - MVT::v12f32, MVT::v16f16, MVT::v16i16, MVT::v16f32, MVT::v16i32, - MVT::v32f32, MVT::v32i32, MVT::v2f64, MVT::v2i64, MVT::v3f64, - MVT::v3i64, MVT::v4f64, MVT::v4i64, MVT::v8f64, MVT::v8i64, - MVT::v16f64, MVT::v16i64, MVT::v32i16, MVT::v32f16}, + {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v4f16, MVT::v4i16, + MVT::v4bf16, MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, + MVT::v4f32, MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, + MVT::v6i32, MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, + MVT::v9f32, MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, + MVT::v11f32, MVT::v12i32, MVT::v12f32, MVT::v16f16, MVT::v16i16, + MVT::v16f32, MVT::v16i32, MVT::v32f32, MVT::v32i32, MVT::v2f64, + MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64, MVT::v4i64, + MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64, MVT::v32i16, + MVT::v32f16}, Custom); setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index a89ef658734b..c84a0934ca81 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -164,6 +164,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, addRegisterClass(MVT::v2bf16, &AMDGPU::SReg_32RegClass); addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass); addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass); + addRegisterClass(MVT::v4bf16, &AMDGPU::SReg_64RegClass); addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass); addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass); addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass); @@ -312,10 +313,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16, - MVT::v4f16, MVT::v3i64, MVT::v3f64, MVT::v6i32, MVT::v6f32, - MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, MVT::v8i16, - MVT::v8f16, MVT::v16i16, MVT::v16f16, MVT::v16i64, MVT::v16f64, - MVT::v32i32, MVT::v32f32, MVT::v32i16, MVT::v32f16}) { + MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32, + MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, + MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16, MVT::v16i64, + MVT::v16f64, MVT::v32i32, MVT::v32f32, MVT::v32i16, MVT::v32f16}) { for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) { switch (Op) { case ISD::LOAD: @@ -421,13 +422,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32}, Expand); - setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16}, Custom); + setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16, MVT::v4bf16}, + Custom); // Avoid stack access for these. // TODO: Generalize to more vector types. setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8, - MVT::v8i8, MVT::v4i16, MVT::v4f16}, + MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16}, Custom); // Deal with vec3 vector operations when widened to vec4. @@ -667,11 +669,15 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32); setOperationAction(ISD::LOAD, MVT::v4f16, Promote); AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32); + setOperationAction(ISD::LOAD, MVT::v4bf16, Promote); + AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32); setOperationAction(ISD::STORE, MVT::v4i16, Promote); AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); setOperationAction(ISD::STORE, MVT::v4f16, Promote); AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); + setOperationAction(ISD::STORE, MVT::v4bf16, Promote); + AddPromotedToType(ISD::STORE, MVT::v4bf16, MVT::v2i32); setOperationAction(ISD::LOAD, MVT::v8i16, Promote); AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32); @@ -781,7 +787,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, Custom); setOperationAction(ISD::FEXP, MVT::v2f16, Custom); - setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16}, Custom); + setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16, MVT::v4bf16}, + Custom); if (Subtarget->hasPackedFP32Ops()) { setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG}, @@ -6805,7 +6812,7 @@ SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op, SDLoc SL(Op); EVT VT = Op.getValueType(); - if (VT == MVT::v4i16 || VT == MVT::v4f16 || + if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16 || VT == MVT::v8i16 || VT == MVT::v8f16) { EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), VT.getVectorNumElements() / 2); @@ -6871,7 +6878,7 @@ SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op, return DAG.getNode(ISD::BITCAST, SL, VT, Blend); } - assert(VT == MVT::v2f16 || VT == MVT::v2i16); + assert(VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16); assert(!Subtarget->hasVOP3PInsts() && "this should be legal"); SDValue Lo = Op.getOperand(0); diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index c29920607967..e8c4d805dbba 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -1548,6 +1548,19 @@ def : BitConvert ; def : BitConvert ; def : BitConvert ; def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; +def : BitConvert ; + // FIXME: Make SGPR def : BitConvert ; diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index 5243262117e7..2a3417e24185 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll @@ -4133,9 +4133,7 @@ define void @test_call_v4bf16(<4 x bfloat> %in, ptr addrspace(5) %out) { ; GFX11-NEXT: v_writelane_b32 v3, s31, 1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-NEXT: scratch_store_b32 v2, v1, off offset:4 dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v2, v0, off dlc +; GFX11-NEXT: scratch_store_b64 v2, v[0:1], off dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: v_readlane_b32 s31, v3, 1 ; GFX11-NEXT: v_readlane_b32 s30, v3, 0 @@ -4397,18 +4395,12 @@ define void @test_call_v8bf16(<8 x bfloat> %in, ptr addrspace(5) %out) { ; GFX11-NEXT: v_writelane_b32 v5, s31, 1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-NEXT: v_add_nc_u32_e32 v6, 12, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_readlane_b32 s31, v5, 1 -; GFX11-NEXT: v_readlane_b32 s30, v5, 0 -; GFX11-NEXT: scratch_store_b32 v6, v3, off dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v4, v2, off offset:8 dlc +; GFX11-NEXT: scratch_store_b64 v4, v[2:3], off offset:8 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v4, v1, off offset:4 dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v4, v0, off dlc +; GFX11-NEXT: scratch_store_b64 v4, v[0:1], off dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_readlane_b32 s31, v5, 1 +; GFX11-NEXT: v_readlane_b32 s30, v5, 0 ; GFX11-NEXT: s_xor_saveexec_b32 s0, -1 ; GFX11-NEXT: scratch_load_b32 v5, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s0 @@ -4759,28 +4751,18 @@ define void @test_call_v16bf16(<16 x bfloat> %in, ptr addrspace(5) %out) { ; GFX11-NEXT: v_writelane_b32 v9, s31, 1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-NEXT: v_add_nc_u32_e32 v10, 28, v8 -; GFX11-NEXT: v_add_nc_u32_e32 v11, 24, v8 -; GFX11-NEXT: v_add_nc_u32_e32 v12, 20, v8 -; GFX11-NEXT: v_add_nc_u32_e32 v13, 12, v8 +; GFX11-NEXT: v_add_nc_u32_e32 v10, 24, v8 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_readlane_b32 s31, v9, 1 -; GFX11-NEXT: scratch_store_b32 v10, v7, off dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v11, v6, off dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v12, v5, off dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v8, v4, off offset:16 dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v13, v3, off dlc +; GFX11-NEXT: v_readlane_b32 s30, v9, 0 +; GFX11-NEXT: scratch_store_b64 v10, v[6:7], off dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v8, v2, off offset:8 dlc +; GFX11-NEXT: scratch_store_b64 v8, v[4:5], off offset:16 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v8, v1, off offset:4 dlc +; GFX11-NEXT: scratch_store_b64 v8, v[2:3], off offset:8 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: scratch_store_b32 v8, v0, off dlc +; GFX11-NEXT: scratch_store_b64 v8, v[0:1], off dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: v_readlane_b32 s30, v9, 0 ; GFX11-NEXT: s_xor_saveexec_b32 s0, -1 ; GFX11-NEXT: scratch_load_b32 v9, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s0 @@ -8691,83 +8673,52 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GFX8-LABEL: v_fadd_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_add_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_add_f32_e32 v3, v4, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_add_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fadd_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_add_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_add_f32_e32 v3, v4, v3 ; GFX9-NEXT: v_add_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_add_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fadd_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_add_f32_e32 v4, v5, v4 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_add_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v3 ; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_fadd_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_add_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_add_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fadd <3 x bfloat> %a, %b ret <3 x bfloat> %op } @@ -8818,82 +8769,81 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GFX8-LABEL: v_fadd_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_add_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_add_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_add_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_add_f32_e32 v3, v5, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fadd_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_add_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_add_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_add_f32_e32 v3, v5, v3 ; GFX9-NEXT: v_add_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_add_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fadd_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_add_f32_e32 v4, v5, v4 +; GFX10-NEXT: v_add_f32_e32 v5, v7, v6 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_add_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fadd_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_add_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_add_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_dual_add_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX11-NEXT: v_add_f32_e32 v5, v7, v6 +; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fadd <4 x bfloat> %a, %b ret <4 x bfloat> %op @@ -8977,138 +8927,138 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GFX8-LABEL: v_fadd_v8bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_add_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX8-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX8-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_add_f32_e32 v7, v9, v7 +; GFX8-NEXT: v_add_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_add_f32_e32 v6, v9, v6 ; GFX8-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX8-NEXT: v_add_f32_e32 v4, v8, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX8-NEXT: v_add_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_add_f32_e32 v2, v2, v5 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX8-NEXT: v_add_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_add_f32_e32 v3, v3, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_add_f32_e32 v0, v0, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v4, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_add_f32_e32 v5, v9, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v7, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fadd_v8bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v4 +; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 +; GFX9-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_add_f32_e32 v7, v9, v7 +; GFX9-NEXT: v_add_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_add_f32_e32 v6, v9, v6 +; GFX9-NEXT: v_add_f32_e32 v1, v1, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4 ; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 +; GFX9-NEXT: v_add_f32_e32 v5, v9, v5 ; GFX9-NEXT: v_add_f32_e32 v0, v0, v4 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_add_f32_e32 v4, v8, v4 -; GFX9-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX9-NEXT: v_add_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_add_f32_e32 v2, v2, v5 -; GFX9-NEXT: v_perm_b32 v2, v2, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX9-NEXT: v_add_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_add_f32_e32 v3, v3, v5 -; GFX9-NEXT: v_perm_b32 v3, v3, v4, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v7, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fadd_v8bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_add_f32_e32 v8, v9, v8 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_add_f32_e32 v9, v11, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX10-NEXT: v_add_f32_e32 v8, v9, v8 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_add_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_add_f32_e32 v4, v11, v10 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_add_f32_e32 v10, v11, v10 +; GFX10-NEXT: v_add_f32_e32 v11, v13, v12 +; GFX10-NEXT: v_add_f32_e32 v0, v0, v4 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX10-NEXT: v_add_f32_e32 v5, v10, v9 ; GFX10-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX10-NEXT: v_add_f32_e32 v6, v12, v11 ; GFX10-NEXT: v_add_f32_e32 v3, v3, v7 -; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fadd_v8bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_add_f32 v9, v11, v10 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_add_f32_e32 v0, v0, v4 ; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_add_f32 v1, v1, v5 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_add_f32_e32 v0, v0, v4 -; GFX11-NEXT: v_add_f32_e32 v4, v11, v10 -; GFX11-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_add_f32_e32 v5, v10, v9 -; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX11-NEXT: v_add_f32_e32 v2, v2, v6 -; GFX11-NEXT: v_add_f32_e32 v6, v12, v11 +; GFX11-NEXT: v_dual_add_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3 ; GFX11-NEXT: v_add_f32_e32 v3, v3, v7 -; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX11-NEXT: v_dual_add_f32 v10, v11, v10 :: v_dual_add_f32 v11, v13, v12 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fadd <8 x bfloat> %a, %b ret <8 x bfloat> %op @@ -9260,252 +9210,254 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX8-LABEL: v_fadd_v16bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_add_f32_e32 v0, v0, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX8-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v1 +; GFX8-NEXT: v_add_f32_e32 v7, v7, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_add_f32_e32 v15, v17, v15 +; GFX8-NEXT: v_add_f32_e32 v6, v6, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_add_f32_e32 v14, v17, v14 +; GFX8-NEXT: v_add_f32_e32 v5, v5, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_add_f32_e32 v13, v17, v13 +; GFX8-NEXT: v_add_f32_e32 v4, v4, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_add_f32_e32 v12, v17, v12 +; GFX8-NEXT: v_add_f32_e32 v3, v3, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_add_f32_e32 v11, v17, v11 +; GFX8-NEXT: v_add_f32_e32 v2, v2, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_add_f32_e32 v10, v17, v10 ; GFX8-NEXT: v_add_f32_e32 v1, v1, v9 -; GFX8-NEXT: v_add_f32_e32 v8, v16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX8-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_add_f32_e32 v2, v2, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX8-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_add_f32_e32 v3, v3, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX8-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_add_f32_e32 v4, v4, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX8-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_add_f32_e32 v5, v5, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX8-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_add_f32_e32 v6, v6, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX8-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_add_f32_e32 v7, v7, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_add_f32_e32 v0, v0, v8 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v8, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_add_f32_e32 v9, v17, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v11, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v12, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v13, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v14, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v15, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fadd_v16bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v8 +; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_add_f32_e32 v7, v7, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_add_f32_e32 v15, v17, v15 +; GFX9-NEXT: v_add_f32_e32 v6, v6, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_add_f32_e32 v14, v17, v14 +; GFX9-NEXT: v_add_f32_e32 v5, v5, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_add_f32_e32 v13, v17, v13 +; GFX9-NEXT: v_add_f32_e32 v4, v4, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_add_f32_e32 v12, v17, v12 +; GFX9-NEXT: v_add_f32_e32 v3, v3, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_add_f32_e32 v11, v17, v11 +; GFX9-NEXT: v_add_f32_e32 v2, v2, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_add_f32_e32 v10, v17, v10 +; GFX9-NEXT: v_add_f32_e32 v1, v1, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8 ; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_add_f32_e32 v9, v17, v9 ; GFX9-NEXT: v_add_f32_e32 v0, v0, v8 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_add_f32_e32 v8, v16, v8 -; GFX9-NEXT: v_add_f32_e32 v1, v1, v9 -; GFX9-NEXT: v_perm_b32 v1, v1, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_add_f32_e32 v2, v2, v9 -; GFX9-NEXT: v_perm_b32 v2, v2, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_add_f32_e32 v3, v3, v9 -; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_add_f32_e32 v4, v4, v9 -; GFX9-NEXT: v_perm_b32 v4, v4, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_add_f32_e32 v5, v5, v9 -; GFX9-NEXT: v_perm_b32 v5, v5, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_add_f32_e32 v6, v6, v9 -; GFX9-NEXT: v_perm_b32 v6, v6, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX9-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_add_f32_e32 v7, v7, v9 -; GFX9-NEXT: v_perm_b32 v7, v7, v8, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v11, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v12, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v13, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v14, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v15, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fadd_v16bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5 ; GFX10-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX10-NEXT: v_add_f32_e32 v7, v7, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX10-NEXT: v_add_f32_e32 v15, v17, v15 +; GFX10-NEXT: v_add_f32_e32 v6, v6, v14 +; GFX10-NEXT: v_add_f32_e32 v14, v19, v18 +; GFX10-NEXT: v_add_f32_e32 v5, v5, v13 +; GFX10-NEXT: v_add_f32_e32 v13, v21, v20 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_add_f32_e32 v17, v18, v17 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_add_f32_e32 v18, v20, v19 ; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX10-NEXT: v_add_f32_e32 v19, v20, v19 +; GFX10-NEXT: v_add_f32_e32 v20, v22, v21 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v8 -; GFX10-NEXT: v_add_f32_e32 v8, v18, v17 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v9 -; GFX10-NEXT: v_add_f32_e32 v9, v20, v19 ; GFX10-NEXT: v_add_f32_e32 v2, v2, v10 -; GFX10-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v11 -; GFX10-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX10-NEXT: v_add_f32_e32 v8, v9, v8 -; GFX10-NEXT: v_add_f32_e32 v3, v3, v10 -; GFX10-NEXT: v_add_f32_e32 v9, v16, v11 +; GFX10-NEXT: v_add_f32_e32 v3, v3, v11 ; GFX10-NEXT: v_add_f32_e32 v4, v4, v12 -; GFX10-NEXT: v_add_f32_e32 v10, v18, v17 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_add_f32_e32 v5, v5, v11 -; GFX10-NEXT: v_add_f32_e32 v11, v13, v12 -; GFX10-NEXT: v_add_f32_e32 v6, v6, v14 -; GFX10-NEXT: v_add_f32_e32 v12, v17, v16 -; GFX10-NEXT: v_add_f32_e32 v7, v7, v15 -; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX10-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fadd_v16bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_add_f32_e32 v2, v2, v10 -; GFX11-NEXT: v_dual_add_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_add_f32 v1, v1, v9 :: v_dual_and_b32 v10, 0xffff0000, v11 -; GFX11-NEXT: v_add_f32_e32 v9, v20, v19 -; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_add_f32 v3, v3, v10 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_add_f32_e32 v0, v0, v8 -; GFX11-NEXT: v_add_f32_e32 v8, v18, v17 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_dual_add_f32 v10, v18, v17 :: v_dual_lshlrev_b32 v17, 16, v7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_dual_add_f32 v4, v4, v12 :: v_dual_and_b32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_add_f32 v9, v16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_add_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX11-NEXT: v_add_f32_e32 v5, v5, v13 +; GFX11-NEXT: v_add_f32_e32 v13, v21, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_dual_add_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_add_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX11-NEXT: v_add_f32_e32 v7, v7, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX11-NEXT: v_add_f32_e32 v5, v5, v11 -; GFX11-NEXT: v_add_f32_e32 v11, v13, v12 -; GFX11-NEXT: v_add_f32_e32 v12, v17, v16 -; GFX11-NEXT: v_dual_add_f32 v6, v6, v14 :: v_dual_add_f32 v7, v7, v15 -; GFX11-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 +; GFX11-NEXT: v_add_f32_e32 v15, v17, v15 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_dual_add_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11 +; GFX11-NEXT: v_add_f32_e32 v14, v19, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 +; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_dual_add_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX11-NEXT: v_add_f32_e32 v3, v3, v11 +; GFX11-NEXT: v_dual_add_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_dual_add_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_add_f32 v19, v20, v19 :: v_dual_add_f32 v20, v22, v21 +; GFX11-NEXT: v_add_f32_e32 v1, v1, v9 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fadd <16 x bfloat> %a, %b ret <16 x bfloat> %op @@ -9913,483 +9865,480 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GFX8-LABEL: v_fadd_v32bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_add_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX8-NEXT: v_add_f32_e32 v31, v32, v31 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v31, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX8-NEXT: v_add_f32_e32 v30, v14, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_add_f32_e32 v14, v32, v14 +; GFX8-NEXT: v_add_f32_e32 v13, v13, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_add_f32_e32 v29, v32, v29 +; GFX8-NEXT: v_add_f32_e32 v12, v12, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_add_f32_e32 v28, v32, v28 +; GFX8-NEXT: v_add_f32_e32 v11, v11, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_add_f32_e32 v27, v32, v27 +; GFX8-NEXT: v_add_f32_e32 v10, v10, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX8-NEXT: v_add_f32_e32 v26, v32, v26 +; GFX8-NEXT: v_add_f32_e32 v9, v9, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX8-NEXT: v_add_f32_e32 v8, v8, v24 +; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX8-NEXT: v_add_f32_e32 v25, v32, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 +; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 +; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16 +; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16 +; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16 +; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16 +; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16 +; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX8-NEXT: v_add_f32_e32 v32, v32, v33 +; GFX8-NEXT: v_add_f32_e32 v15, v15, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_add_f32_e32 v24, v33, v24 +; GFX8-NEXT: v_add_f32_e32 v7, v7, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_add_f32_e32 v23, v33, v23 +; GFX8-NEXT: v_add_f32_e32 v6, v6, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_add_f32_e32 v22, v33, v22 +; GFX8-NEXT: v_add_f32_e32 v5, v5, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_add_f32_e32 v21, v33, v21 +; GFX8-NEXT: v_add_f32_e32 v4, v4, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_add_f32_e32 v20, v33, v20 +; GFX8-NEXT: v_add_f32_e32 v3, v3, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_add_f32_e32 v19, v33, v19 +; GFX8-NEXT: v_add_f32_e32 v2, v2, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_add_f32_e32 v18, v33, v18 ; GFX8-NEXT: v_add_f32_e32 v1, v1, v17 -; GFX8-NEXT: v_add_f32_e32 v16, v31, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_add_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_add_f32_e32 v17, v33, v17 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX8-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_add_f32_e32 v2, v2, v17 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX8-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_add_f32_e32 v3, v3, v17 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX8-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_add_f32_e32 v4, v4, v17 -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v21 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_add_f32_e32 v5, v5, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v6 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v22 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_add_f32_e32 v6, v6, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v7 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v23 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_add_f32_e32 v7, v7, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v24 -; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_add_f32_e32 v8, v8, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_alignbit_b32 v8, v8, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v25 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX8-NEXT: v_add_f32_e32 v9, v9, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 -; GFX8-NEXT: v_alignbit_b32 v9, v9, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v10 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v26 -; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX8-NEXT: v_add_f32_e32 v10, v10, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX8-NEXT: v_alignbit_b32 v10, v10, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v11 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v27 -; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX8-NEXT: v_add_f32_e32 v11, v11, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_alignbit_b32 v11, v11, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v12 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v28 -; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX8-NEXT: v_add_f32_e32 v12, v12, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_alignbit_b32 v12, v12, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v29 -; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX8-NEXT: v_add_f32_e32 v13, v13, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_alignbit_b32 v13, v13, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v14 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v30 -; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX8-NEXT: v_add_f32_e32 v14, v14, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_alignbit_b32 v14, v14, v16, 16 -; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX8-NEXT: v_add_f32_e32 v15, v15, v17 -; GFX8-NEXT: v_add_f32_e32 v16, v18, v16 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_alignbit_b32 v15, v15, v16, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v20, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v21, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16 +; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16 +; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fadd_v32bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX9-NEXT: v_add_f32_e32 v31, v32, v31 -; GFX9-NEXT: v_add_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_add_f32_e32 v14, v14, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_add_f32_e32 v30, v32, v30 +; GFX9-NEXT: v_add_f32_e32 v13, v13, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_add_f32_e32 v29, v32, v29 +; GFX9-NEXT: v_add_f32_e32 v12, v12, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_add_f32_e32 v28, v32, v28 +; GFX9-NEXT: v_add_f32_e32 v11, v11, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_add_f32_e32 v27, v32, v27 +; GFX9-NEXT: v_add_f32_e32 v10, v10, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_add_f32_e32 v26, v32, v26 +; GFX9-NEXT: v_add_f32_e32 v9, v9, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX9-NEXT: v_add_f32_e32 v8, v8, v24 +; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX9-NEXT: v_add_f32_e32 v25, v32, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v31, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4 +; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4 +; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4 +; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4 +; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4 +; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4 +; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_add_f32_e32 v32, v32, v33 +; GFX9-NEXT: v_add_f32_e32 v15, v15, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_add_f32_e32 v24, v33, v24 +; GFX9-NEXT: v_add_f32_e32 v7, v7, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_add_f32_e32 v23, v33, v23 +; GFX9-NEXT: v_add_f32_e32 v6, v6, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_add_f32_e32 v22, v33, v22 +; GFX9-NEXT: v_add_f32_e32 v5, v5, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_add_f32_e32 v21, v33, v21 +; GFX9-NEXT: v_add_f32_e32 v4, v4, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_add_f32_e32 v20, v33, v20 +; GFX9-NEXT: v_add_f32_e32 v3, v3, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_add_f32_e32 v19, v33, v19 +; GFX9-NEXT: v_add_f32_e32 v2, v2, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_add_f32_e32 v16, v31, v16 +; GFX9-NEXT: v_add_f32_e32 v18, v33, v18 ; GFX9-NEXT: v_add_f32_e32 v1, v1, v17 -; GFX9-NEXT: v_perm_b32 v1, v1, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_add_f32_e32 v2, v2, v17 -; GFX9-NEXT: v_perm_b32 v2, v2, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_add_f32_e32 v3, v3, v17 -; GFX9-NEXT: v_perm_b32 v3, v3, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_add_f32_e32 v4, v4, v17 -; GFX9-NEXT: v_perm_b32 v4, v4, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v21 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_add_f32_e32 v5, v5, v17 -; GFX9-NEXT: v_perm_b32 v5, v5, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v22 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_add_f32_e32 v6, v6, v17 -; GFX9-NEXT: v_perm_b32 v6, v6, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v23 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_add_f32_e32 v7, v7, v17 -; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v8 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v24 -; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX9-NEXT: v_add_f32_e32 v8, v8, v17 -; GFX9-NEXT: v_perm_b32 v8, v8, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v25 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_add_f32_e32 v9, v9, v17 -; GFX9-NEXT: v_perm_b32 v9, v9, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v26 -; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX9-NEXT: v_add_f32_e32 v10, v10, v17 -; GFX9-NEXT: v_perm_b32 v10, v10, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v27 -; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX9-NEXT: v_add_f32_e32 v11, v11, v17 -; GFX9-NEXT: v_perm_b32 v11, v11, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v28 -; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX9-NEXT: v_add_f32_e32 v12, v12, v17 -; GFX9-NEXT: v_perm_b32 v12, v12, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v29 -; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX9-NEXT: v_add_f32_e32 v13, v13, v17 -; GFX9-NEXT: v_perm_b32 v13, v13, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v14 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v30 -; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX9-NEXT: v_add_f32_e32 v14, v14, v17 -; GFX9-NEXT: v_perm_b32 v14, v14, v16, s4 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v15 -; GFX9-NEXT: v_add_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX9-NEXT: v_add_f32_e32 v15, v15, v17 -; GFX9-NEXT: v_perm_b32 v15, v15, v16, s4 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_add_f32_e32 v17, v33, v17 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v20, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v21, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4 +; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fadd_v32bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v22 -; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10 +; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13 +; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_add_f32_e32 v39, v48, v39 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17 +; GFX10-NEXT: v_add_f32_e32 v11, v11, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1 ; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_add_f32_e32 v5, v5, v21 -; GFX10-NEXT: v_add_f32_e32 v21, v53, v52 -; GFX10-NEXT: v_add_f32_e32 v6, v6, v22 -; GFX10-NEXT: v_add_f32_e32 v22, v55, v54 -; GFX10-NEXT: v_add_f32_e32 v7, v7, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v25 -; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v9 +; GFX10-NEXT: v_add_f32_e32 v49, v50, v49 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16 +; GFX10-NEXT: v_add_f32_e32 v10, v10, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9 ; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX10-NEXT: v_add_f32_e32 v32, v33, v32 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v10 -; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_add_f32_e32 v33, v34, v33 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20 +; GFX10-NEXT: v_add_f32_e32 v14, v14, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4 +; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_add_f32_e32 v35, v36, v35 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19 +; GFX10-NEXT: v_add_f32_e32 v13, v13, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3 +; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_add_f32_e32 v37, v38, v37 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18 +; GFX10-NEXT: v_add_f32_e32 v12, v12, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX10-NEXT: v_add_f32_e32 v34, v35, v34 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v11 -; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v28 -; GFX10-NEXT: v_add_f32_e32 v36, v37, v36 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v12 -; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_add_f32_e32 v51, v52, v51 +; GFX10-NEXT: v_add_f32_e32 v9, v9, v25 +; GFX10-NEXT: v_add_f32_e32 v25, v54, v53 +; GFX10-NEXT: v_add_f32_e32 v8, v8, v24 +; GFX10-NEXT: v_add_f32_e32 v24, v64, v55 +; GFX10-NEXT: v_add_f32_e32 v7, v7, v23 +; GFX10-NEXT: v_add_f32_e32 v23, v66, v65 +; GFX10-NEXT: v_add_f32_e32 v6, v6, v22 +; GFX10-NEXT: v_add_f32_e32 v22, v68, v67 +; GFX10-NEXT: v_add_f32_e32 v5, v5, v21 +; GFX10-NEXT: v_add_f32_e32 v21, v30, v34 +; GFX10-NEXT: v_add_f32_e32 v29, v29, v36 +; GFX10-NEXT: v_add_f32_e32 v28, v28, v38 +; GFX10-NEXT: v_add_f32_e32 v27, v27, v48 +; GFX10-NEXT: v_add_f32_e32 v26, v26, v50 ; GFX10-NEXT: v_add_f32_e32 v2, v2, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v29 -; GFX10-NEXT: v_add_f32_e32 v38, v39, v38 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v13 -; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX10-NEXT: v_add_f32_e32 v3, v3, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v30 -; GFX10-NEXT: v_add_f32_e32 v48, v49, v48 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v14 -; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX10-NEXT: v_add_f32_e32 v4, v4, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v15 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX10-NEXT: v_add_f32_e32 v50, v51, v50 -; GFX10-NEXT: v_add_f32_e32 v23, v65, v64 -; GFX10-NEXT: v_add_f32_e32 v8, v8, v24 -; GFX10-NEXT: v_add_f32_e32 v24, v67, v66 -; GFX10-NEXT: v_add_f32_e32 v9, v9, v25 -; GFX10-NEXT: v_add_f32_e32 v25, v33, v68 -; GFX10-NEXT: v_add_f32_e32 v10, v10, v26 -; GFX10-NEXT: v_add_f32_e32 v16, v35, v16 -; GFX10-NEXT: v_add_f32_e32 v11, v11, v27 -; GFX10-NEXT: v_add_f32_e32 v17, v37, v17 -; GFX10-NEXT: v_add_f32_e32 v12, v12, v28 -; GFX10-NEXT: v_add_f32_e32 v18, v39, v18 -; GFX10-NEXT: v_add_f32_e32 v13, v13, v29 -; GFX10-NEXT: v_add_f32_e32 v19, v49, v19 -; GFX10-NEXT: v_add_f32_e32 v14, v14, v30 -; GFX10-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v34, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v36, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v38, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v48, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v50, 0x7060302 -; GFX10-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX10-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX10-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX10-NEXT: v_perm_b32 v11, v11, v16, 0x7060302 -; GFX10-NEXT: v_perm_b32 v12, v12, v17, 0x7060302 -; GFX10-NEXT: v_perm_b32 v13, v13, v18, 0x7060302 -; GFX10-NEXT: v_perm_b32 v14, v14, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302 +; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302 +; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302 +; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302 +; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302 +; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v31 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v31 -; GFX10-NEXT: v_add_f32_e32 v20, v20, v21 -; GFX10-NEXT: v_add_f32_e32 v15, v15, v22 -; GFX10-NEXT: v_perm_b32 v15, v15, v20, 0x7060302 +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31 +; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX10-NEXT: v_add_f32_e32 v16, v32, v16 +; GFX10-NEXT: v_add_f32_e32 v15, v15, v17 +; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fadd_v32bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v10 -; GFX11-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v27 -; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX11-NEXT: v_dual_add_f32 v10, v10, v26 :: v_dual_and_b32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v25 +; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v17 +; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16 +; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX11-NEXT: v_dual_add_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4 +; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v28 -; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v22 +; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX11-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24 +; GFX11-NEXT: v_dual_add_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10 +; GFX11-NEXT: v_dual_add_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_dual_add_f32 v2, v2, v18 :: v_dual_add_f32 v3, v3, v19 +; GFX11-NEXT: v_dual_add_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26 +; GFX11-NEXT: v_dual_add_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26 +; GFX11-NEXT: v_add_f32_e32 v6, v6, v22 +; GFX11-NEXT: v_dual_add_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28 ; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v29 -; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v13 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_add_f32_e32 v10, v10, v26 +; GFX11-NEXT: v_add_f32_e32 v26, v52, v51 +; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX11-NEXT: v_add_f32_e32 v25, v54, v53 +; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 ; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13 ; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v30 -; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX11-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v15 -; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX11-NEXT: v_dual_add_f32 v11, v11, v27 :: v_dual_and_b32 v20, 0xffff0000, v20 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_dual_add_f32 v26, v71, v70 :: v_dual_lshlrev_b32 v49, 16, v4 -; GFX11-NEXT: v_dual_add_f32 v13, v13, v29 :: v_dual_and_b32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_add_f32_e32 v4, v4, v20 -; GFX11-NEXT: v_dual_add_f32 v8, v8, v24 :: v_dual_add_f32 v9, v9, v25 -; GFX11-NEXT: v_add_f32_e32 v25, v69, v68 -; GFX11-NEXT: v_dual_add_f32 v20, v51, v50 :: v_dual_lshlrev_b32 v39, 16, v3 -; GFX11-NEXT: v_add_f32_e32 v27, v81, v80 -; GFX11-NEXT: v_add_f32_e32 v12, v12, v28 -; GFX11-NEXT: v_dual_add_f32 v28, v83, v82 :: v_dual_add_f32 v29, v85, v84 -; GFX11-NEXT: v_dual_add_f32 v6, v6, v22 :: v_dual_and_b32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_add_f32_e32 v22, v55, v54 -; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX11-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_dual_add_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27 +; GFX11-NEXT: v_add_f32_e32 v24, v64, v55 +; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX11-NEXT: v_add_f32_e32 v7, v7, v23 +; GFX11-NEXT: v_add_f32_e32 v23, v66, v65 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_add_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_dual_add_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30 +; GFX11-NEXT: v_dual_add_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX11-NEXT: v_dual_add_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_dual_add_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30 +; GFX11-NEXT: v_add_f32_e32 v29, v38, v37 +; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_add_f32_e32 v37, v86, v85 +; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 ; GFX11-NEXT: v_add_f32_e32 v14, v14, v30 -; GFX11-NEXT: v_dual_add_f32 v7, v7, v23 :: v_dual_and_b32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_add_f32_e32 v23, v65, v64 -; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX11-NEXT: v_dual_add_f32 v24, v67, v66 :: v_dual_and_b32 v21, 0xffff0000, v21 -; GFX11-NEXT: v_add_f32_e32 v2, v2, v18 -; GFX11-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v32, 16, v16 -; GFX11-NEXT: v_add_f32_e32 v18, v39, v38 -; GFX11-NEXT: v_dual_add_f32 v3, v3, v19 :: v_dual_and_b32 v16, 0xffff0000, v16 -; GFX11-NEXT: v_add_f32_e32 v19, v49, v48 -; GFX11-NEXT: v_add_f32_e32 v17, v37, v36 -; GFX11-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX11-NEXT: v_dual_add_f32 v5, v5, v21 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_add_f32_e32 v21, v53, v52 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v2, v2, v17, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v18, 0x7060302 -; GFX11-NEXT: v_add_f32_e32 v0, v0, v16 -; GFX11-NEXT: v_add_f32_e32 v16, v35, v34 -; GFX11-NEXT: v_add_f32_e32 v32, v33, v32 -; GFX11-NEXT: v_perm_b32 v4, v4, v19, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v20, 0x7060302 -; GFX11-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX11-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX11-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX11-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX11-NEXT: v_perm_b32 v11, v11, v26, 0x7060302 -; GFX11-NEXT: v_perm_b32 v12, v12, v27, 0x7060302 -; GFX11-NEXT: v_perm_b32 v13, v13, v28, 0x7060302 -; GFX11-NEXT: v_perm_b32 v14, v14, v29, 0x7060302 -; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_add_f32 v16, v86, v16 :: v_dual_and_b32 v17, 0xffff0000, v31 -; GFX11-NEXT: v_add_f32_e32 v15, v15, v17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_add_f32 v30, v36, v35 :: v_dual_add_f32 v33, v34, v33 +; GFX11-NEXT: v_dual_add_f32 v34, v80, v71 :: v_dual_add_f32 v35, v82, v81 +; GFX11-NEXT: v_add_f32_e32 v36, v84, v83 +; GFX11-NEXT: v_dual_add_f32 v16, v32, v16 :: v_dual_add_f32 v15, v15, v17 +; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302 +; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302 +; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302 +; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302 +; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302 +; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302 +; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fadd <32 x bfloat> %a, %b @@ -10681,83 +10630,52 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GFX8-LABEL: v_fsub_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_sub_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_sub_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_sub_f32_e32 v3, v4, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_sub_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_sub_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fsub_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_sub_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_sub_f32_e32 v3, v4, v3 ; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_sub_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fsub_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_sub_f32_e32 v4, v5, v4 ; GFX10-NEXT: v_sub_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_sub_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_sub_f32_e32 v1, v1, v3 ; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_fsub_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_sub_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_sub_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fsub <3 x bfloat> %a, %b ret <3 x bfloat> %op } @@ -10808,82 +10726,81 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GFX8-LABEL: v_fsub_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_sub_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_sub_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_sub_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_sub_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_sub_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_sub_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fsub_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_sub_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_sub_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_sub_f32_e32 v3, v5, v3 ; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_sub_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_sub_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fsub_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_sub_f32_e32 v4, v5, v4 +; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6 ; GFX10-NEXT: v_sub_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_sub_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_sub_f32_e32 v1, v1, v3 -; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fsub_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_sub_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_sub_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_dual_sub_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_sub_f32_e32 v1, v1, v3 +; GFX11-NEXT: v_sub_f32_e32 v5, v7, v6 +; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fsub <4 x bfloat> %a, %b ret <4 x bfloat> %op @@ -11068,83 +10985,52 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GFX8-LABEL: v_fmul_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_mul_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_mul_f32_e32 v3, v4, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_mul_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fmul_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_mul_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3 ; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_mul_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fmul_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_mul_f32_e32 v4, v5, v4 ; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_mul_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3 ; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_fmul_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_mul_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_mul_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_mul_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fmul <3 x bfloat> %a, %b ret <3 x bfloat> %op } @@ -11195,82 +11081,81 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GFX8-LABEL: v_fmul_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_mul_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_mul_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_mul_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fmul_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_mul_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_mul_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_mul_f32_e32 v3, v5, v3 ; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_mul_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fmul_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_mul_f32_e32 v4, v5, v4 +; GFX10-NEXT: v_mul_f32_e32 v5, v7, v6 ; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_mul_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fmul_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_mul_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_mul_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_mul_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_dual_mul_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX11-NEXT: v_mul_f32_e32 v5, v7, v6 +; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fmul <4 x bfloat> %a, %b ret <4 x bfloat> %op @@ -11354,138 +11239,138 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GFX8-LABEL: v_fmul_v8bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_mul_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX8-NEXT: v_mul_f32_e32 v3, v3, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_mul_f32_e32 v7, v9, v7 +; GFX8-NEXT: v_mul_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_mul_f32_e32 v6, v9, v6 ; GFX8-NEXT: v_mul_f32_e32 v1, v1, v5 -; GFX8-NEXT: v_mul_f32_e32 v4, v8, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX8-NEXT: v_mul_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_mul_f32_e32 v2, v2, v5 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX8-NEXT: v_mul_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_mul_f32_e32 v3, v3, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, v0, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v4, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_mul_f32_e32 v5, v9, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v7, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fmul_v8bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v4 +; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 +; GFX9-NEXT: v_mul_f32_e32 v3, v3, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_mul_f32_e32 v7, v9, v7 +; GFX9-NEXT: v_mul_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_mul_f32_e32 v6, v9, v6 +; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4 ; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 +; GFX9-NEXT: v_mul_f32_e32 v5, v9, v5 ; GFX9-NEXT: v_mul_f32_e32 v0, v0, v4 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_mul_f32_e32 v4, v8, v4 -; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5 -; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX9-NEXT: v_mul_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_mul_f32_e32 v2, v2, v5 -; GFX9-NEXT: v_perm_b32 v2, v2, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX9-NEXT: v_mul_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_mul_f32_e32 v3, v3, v5 -; GFX9-NEXT: v_perm_b32 v3, v3, v4, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v7, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fmul_v8bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_mul_f32_e32 v8, v9, v8 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_mul_f32_e32 v9, v11, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX10-NEXT: v_mul_f32_e32 v8, v9, v8 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_mul_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_mul_f32_e32 v4, v11, v10 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_mul_f32_e32 v10, v11, v10 +; GFX10-NEXT: v_mul_f32_e32 v11, v13, v12 +; GFX10-NEXT: v_mul_f32_e32 v0, v0, v4 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v5 -; GFX10-NEXT: v_mul_f32_e32 v5, v10, v9 ; GFX10-NEXT: v_mul_f32_e32 v2, v2, v6 -; GFX10-NEXT: v_mul_f32_e32 v6, v12, v11 ; GFX10-NEXT: v_mul_f32_e32 v3, v3, v7 -; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fmul_v8bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_mul_f32 v9, v11, v10 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_f32_e32 v0, v0, v4 ; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_mul_f32 v1, v1, v5 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_mul_f32_e32 v0, v0, v4 -; GFX11-NEXT: v_mul_f32_e32 v4, v11, v10 -; GFX11-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_mul_f32_e32 v5, v10, v9 -; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX11-NEXT: v_mul_f32_e32 v2, v2, v6 -; GFX11-NEXT: v_mul_f32_e32 v6, v12, v11 +; GFX11-NEXT: v_dual_mul_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_dual_mul_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3 ; GFX11-NEXT: v_mul_f32_e32 v3, v3, v7 -; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX11-NEXT: v_dual_mul_f32 v10, v11, v10 :: v_dual_mul_f32 v11, v13, v12 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fmul <8 x bfloat> %a, %b ret <8 x bfloat> %op @@ -11637,252 +11522,254 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX8-LABEL: v_fmul_v16bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_mul_f32_e32 v0, v0, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX8-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v1 +; GFX8-NEXT: v_mul_f32_e32 v7, v7, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_mul_f32_e32 v15, v17, v15 +; GFX8-NEXT: v_mul_f32_e32 v6, v6, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_mul_f32_e32 v14, v17, v14 +; GFX8-NEXT: v_mul_f32_e32 v5, v5, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_mul_f32_e32 v13, v17, v13 +; GFX8-NEXT: v_mul_f32_e32 v4, v4, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_mul_f32_e32 v12, v17, v12 +; GFX8-NEXT: v_mul_f32_e32 v3, v3, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_mul_f32_e32 v11, v17, v11 +; GFX8-NEXT: v_mul_f32_e32 v2, v2, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_mul_f32_e32 v10, v17, v10 ; GFX8-NEXT: v_mul_f32_e32 v1, v1, v9 -; GFX8-NEXT: v_mul_f32_e32 v8, v16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_mul_f32_e32 v2, v2, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_mul_f32_e32 v3, v3, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_mul_f32_e32 v4, v4, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_mul_f32_e32 v5, v5, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_mul_f32_e32 v6, v6, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX8-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_mul_f32_e32 v7, v7, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, v0, v8 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v8, 16 -; GFX8-NEXT: s_setpc_b64 s[30:31] -; +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_mul_f32_e32 v9, v17, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v11, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v12, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v13, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v14, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v15, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 +; GFX8-NEXT: s_setpc_b64 s[30:31] +; ; GFX9-LABEL: v_fmul_v16bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v8 +; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_mul_f32_e32 v7, v7, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_mul_f32_e32 v15, v17, v15 +; GFX9-NEXT: v_mul_f32_e32 v6, v6, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_mul_f32_e32 v14, v17, v14 +; GFX9-NEXT: v_mul_f32_e32 v5, v5, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_mul_f32_e32 v13, v17, v13 +; GFX9-NEXT: v_mul_f32_e32 v4, v4, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_mul_f32_e32 v12, v17, v12 +; GFX9-NEXT: v_mul_f32_e32 v3, v3, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_mul_f32_e32 v11, v17, v11 +; GFX9-NEXT: v_mul_f32_e32 v2, v2, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_mul_f32_e32 v10, v17, v10 +; GFX9-NEXT: v_mul_f32_e32 v1, v1, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8 ; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_mul_f32_e32 v9, v17, v9 ; GFX9-NEXT: v_mul_f32_e32 v0, v0, v8 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_mul_f32_e32 v8, v16, v8 -; GFX9-NEXT: v_mul_f32_e32 v1, v1, v9 -; GFX9-NEXT: v_perm_b32 v1, v1, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_mul_f32_e32 v2, v2, v9 -; GFX9-NEXT: v_perm_b32 v2, v2, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_mul_f32_e32 v3, v3, v9 -; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_mul_f32_e32 v4, v4, v9 -; GFX9-NEXT: v_perm_b32 v4, v4, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_mul_f32_e32 v5, v5, v9 -; GFX9-NEXT: v_perm_b32 v5, v5, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_mul_f32_e32 v6, v6, v9 -; GFX9-NEXT: v_perm_b32 v6, v6, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX9-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_mul_f32_e32 v7, v7, v9 -; GFX9-NEXT: v_perm_b32 v7, v7, v8, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v11, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v12, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v13, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v14, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v15, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fmul_v16bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5 ; GFX10-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX10-NEXT: v_mul_f32_e32 v7, v7, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX10-NEXT: v_mul_f32_e32 v15, v17, v15 +; GFX10-NEXT: v_mul_f32_e32 v6, v6, v14 +; GFX10-NEXT: v_mul_f32_e32 v14, v19, v18 +; GFX10-NEXT: v_mul_f32_e32 v5, v5, v13 +; GFX10-NEXT: v_mul_f32_e32 v13, v21, v20 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_mul_f32_e32 v17, v18, v17 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_mul_f32_e32 v18, v20, v19 ; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX10-NEXT: v_mul_f32_e32 v19, v20, v19 +; GFX10-NEXT: v_mul_f32_e32 v20, v22, v21 ; GFX10-NEXT: v_mul_f32_e32 v0, v0, v8 -; GFX10-NEXT: v_mul_f32_e32 v8, v18, v17 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v9 -; GFX10-NEXT: v_mul_f32_e32 v9, v20, v19 ; GFX10-NEXT: v_mul_f32_e32 v2, v2, v10 -; GFX10-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v11 -; GFX10-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX10-NEXT: v_mul_f32_e32 v8, v9, v8 -; GFX10-NEXT: v_mul_f32_e32 v3, v3, v10 -; GFX10-NEXT: v_mul_f32_e32 v9, v16, v11 +; GFX10-NEXT: v_mul_f32_e32 v3, v3, v11 ; GFX10-NEXT: v_mul_f32_e32 v4, v4, v12 -; GFX10-NEXT: v_mul_f32_e32 v10, v18, v17 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_mul_f32_e32 v5, v5, v11 -; GFX10-NEXT: v_mul_f32_e32 v11, v13, v12 -; GFX10-NEXT: v_mul_f32_e32 v6, v6, v14 -; GFX10-NEXT: v_mul_f32_e32 v12, v17, v16 -; GFX10-NEXT: v_mul_f32_e32 v7, v7, v15 -; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX10-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fmul_v16bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_mul_f32_e32 v2, v2, v10 -; GFX11-NEXT: v_dual_mul_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_mul_f32 v1, v1, v9 :: v_dual_and_b32 v10, 0xffff0000, v11 -; GFX11-NEXT: v_mul_f32_e32 v9, v20, v19 -; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_mul_f32 v3, v3, v10 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_mul_f32_e32 v0, v0, v8 -; GFX11-NEXT: v_mul_f32_e32 v8, v18, v17 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_dual_mul_f32 v10, v18, v17 :: v_dual_lshlrev_b32 v17, 16, v7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_dual_mul_f32 v4, v4, v12 :: v_dual_and_b32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_mul_f32 v9, v16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_mul_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX11-NEXT: v_mul_f32_e32 v5, v5, v13 +; GFX11-NEXT: v_mul_f32_e32 v13, v21, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_dual_mul_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_mul_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX11-NEXT: v_mul_f32_e32 v7, v7, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX11-NEXT: v_mul_f32_e32 v5, v5, v11 -; GFX11-NEXT: v_mul_f32_e32 v11, v13, v12 -; GFX11-NEXT: v_mul_f32_e32 v12, v17, v16 -; GFX11-NEXT: v_dual_mul_f32 v6, v6, v14 :: v_dual_mul_f32 v7, v7, v15 -; GFX11-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 +; GFX11-NEXT: v_mul_f32_e32 v15, v17, v15 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_dual_mul_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11 +; GFX11-NEXT: v_mul_f32_e32 v14, v19, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 +; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_dual_mul_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX11-NEXT: v_mul_f32_e32 v3, v3, v11 +; GFX11-NEXT: v_dual_mul_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_dual_mul_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_mul_f32 v19, v20, v19 :: v_dual_mul_f32 v20, v22, v21 +; GFX11-NEXT: v_mul_f32_e32 v1, v1, v9 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fmul <16 x bfloat> %a, %b ret <16 x bfloat> %op @@ -12290,483 +12177,480 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GFX8-LABEL: v_fmul_v32bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_mul_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX8-NEXT: v_mul_f32_e32 v31, v32, v31 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v31, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX8-NEXT: v_mul_f32_e32 v30, v14, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_mul_f32_e32 v14, v32, v14 +; GFX8-NEXT: v_mul_f32_e32 v13, v13, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_mul_f32_e32 v29, v32, v29 +; GFX8-NEXT: v_mul_f32_e32 v12, v12, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_mul_f32_e32 v28, v32, v28 +; GFX8-NEXT: v_mul_f32_e32 v11, v11, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_mul_f32_e32 v27, v32, v27 +; GFX8-NEXT: v_mul_f32_e32 v10, v10, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX8-NEXT: v_mul_f32_e32 v26, v32, v26 +; GFX8-NEXT: v_mul_f32_e32 v9, v9, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX8-NEXT: v_mul_f32_e32 v8, v8, v24 +; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX8-NEXT: v_mul_f32_e32 v25, v32, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 +; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 +; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16 +; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16 +; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16 +; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16 +; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16 +; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX8-NEXT: v_mul_f32_e32 v32, v32, v33 +; GFX8-NEXT: v_mul_f32_e32 v15, v15, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_mul_f32_e32 v24, v33, v24 +; GFX8-NEXT: v_mul_f32_e32 v7, v7, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_mul_f32_e32 v23, v33, v23 +; GFX8-NEXT: v_mul_f32_e32 v6, v6, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_mul_f32_e32 v22, v33, v22 +; GFX8-NEXT: v_mul_f32_e32 v5, v5, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_mul_f32_e32 v21, v33, v21 +; GFX8-NEXT: v_mul_f32_e32 v4, v4, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_mul_f32_e32 v20, v33, v20 +; GFX8-NEXT: v_mul_f32_e32 v3, v3, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_mul_f32_e32 v19, v33, v19 +; GFX8-NEXT: v_mul_f32_e32 v2, v2, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_mul_f32_e32 v18, v33, v18 ; GFX8-NEXT: v_mul_f32_e32 v1, v1, v17 -; GFX8-NEXT: v_mul_f32_e32 v16, v31, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_mul_f32_e32 v17, v33, v17 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX8-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_mul_f32_e32 v2, v2, v17 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX8-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_mul_f32_e32 v3, v3, v17 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX8-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_mul_f32_e32 v4, v4, v17 -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v21 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_mul_f32_e32 v5, v5, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v6 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v22 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_mul_f32_e32 v6, v6, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v7 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v23 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_mul_f32_e32 v7, v7, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v24 -; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_mul_f32_e32 v8, v8, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_alignbit_b32 v8, v8, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v25 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX8-NEXT: v_mul_f32_e32 v9, v9, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 -; GFX8-NEXT: v_alignbit_b32 v9, v9, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v10 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v26 -; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX8-NEXT: v_mul_f32_e32 v10, v10, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX8-NEXT: v_alignbit_b32 v10, v10, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v11 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v27 -; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX8-NEXT: v_mul_f32_e32 v11, v11, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_alignbit_b32 v11, v11, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v12 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v28 -; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX8-NEXT: v_mul_f32_e32 v12, v12, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_alignbit_b32 v12, v12, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v29 -; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX8-NEXT: v_mul_f32_e32 v13, v13, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_alignbit_b32 v13, v13, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v14 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v30 -; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX8-NEXT: v_mul_f32_e32 v14, v14, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_alignbit_b32 v14, v14, v16, 16 -; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX8-NEXT: v_mul_f32_e32 v15, v15, v17 -; GFX8-NEXT: v_mul_f32_e32 v16, v18, v16 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_alignbit_b32 v15, v15, v16, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v20, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v21, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16 +; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16 +; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fmul_v32bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX9-NEXT: v_mul_f32_e32 v31, v32, v31 -; GFX9-NEXT: v_mul_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_mul_f32_e32 v14, v14, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_mul_f32_e32 v30, v32, v30 +; GFX9-NEXT: v_mul_f32_e32 v13, v13, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_mul_f32_e32 v29, v32, v29 +; GFX9-NEXT: v_mul_f32_e32 v12, v12, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_mul_f32_e32 v28, v32, v28 +; GFX9-NEXT: v_mul_f32_e32 v11, v11, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_mul_f32_e32 v27, v32, v27 +; GFX9-NEXT: v_mul_f32_e32 v10, v10, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_mul_f32_e32 v26, v32, v26 +; GFX9-NEXT: v_mul_f32_e32 v9, v9, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX9-NEXT: v_mul_f32_e32 v8, v8, v24 +; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX9-NEXT: v_mul_f32_e32 v25, v32, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v31, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4 +; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4 +; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4 +; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4 +; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4 +; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4 +; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_mul_f32_e32 v32, v32, v33 +; GFX9-NEXT: v_mul_f32_e32 v15, v15, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_mul_f32_e32 v24, v33, v24 +; GFX9-NEXT: v_mul_f32_e32 v7, v7, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_mul_f32_e32 v23, v33, v23 +; GFX9-NEXT: v_mul_f32_e32 v6, v6, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_mul_f32_e32 v22, v33, v22 +; GFX9-NEXT: v_mul_f32_e32 v5, v5, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_mul_f32_e32 v21, v33, v21 +; GFX9-NEXT: v_mul_f32_e32 v4, v4, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_mul_f32_e32 v20, v33, v20 +; GFX9-NEXT: v_mul_f32_e32 v3, v3, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_mul_f32_e32 v19, v33, v19 +; GFX9-NEXT: v_mul_f32_e32 v2, v2, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_mul_f32_e32 v16, v31, v16 +; GFX9-NEXT: v_mul_f32_e32 v18, v33, v18 ; GFX9-NEXT: v_mul_f32_e32 v1, v1, v17 -; GFX9-NEXT: v_perm_b32 v1, v1, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_mul_f32_e32 v2, v2, v17 -; GFX9-NEXT: v_perm_b32 v2, v2, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_mul_f32_e32 v3, v3, v17 -; GFX9-NEXT: v_perm_b32 v3, v3, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_mul_f32_e32 v4, v4, v17 -; GFX9-NEXT: v_perm_b32 v4, v4, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v21 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_mul_f32_e32 v5, v5, v17 -; GFX9-NEXT: v_perm_b32 v5, v5, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v22 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_mul_f32_e32 v6, v6, v17 -; GFX9-NEXT: v_perm_b32 v6, v6, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v23 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_mul_f32_e32 v7, v7, v17 -; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v8 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v24 -; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX9-NEXT: v_mul_f32_e32 v8, v8, v17 -; GFX9-NEXT: v_perm_b32 v8, v8, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v25 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_mul_f32_e32 v9, v9, v17 -; GFX9-NEXT: v_perm_b32 v9, v9, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v26 -; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX9-NEXT: v_mul_f32_e32 v10, v10, v17 -; GFX9-NEXT: v_perm_b32 v10, v10, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v27 -; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX9-NEXT: v_mul_f32_e32 v11, v11, v17 -; GFX9-NEXT: v_perm_b32 v11, v11, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v28 -; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX9-NEXT: v_mul_f32_e32 v12, v12, v17 -; GFX9-NEXT: v_perm_b32 v12, v12, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v29 -; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX9-NEXT: v_mul_f32_e32 v13, v13, v17 -; GFX9-NEXT: v_perm_b32 v13, v13, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v14 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v30 -; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX9-NEXT: v_mul_f32_e32 v14, v14, v17 -; GFX9-NEXT: v_perm_b32 v14, v14, v16, s4 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v15 -; GFX9-NEXT: v_mul_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX9-NEXT: v_mul_f32_e32 v15, v15, v17 -; GFX9-NEXT: v_perm_b32 v15, v15, v16, s4 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_mul_f32_e32 v17, v33, v17 +; GFX9-NEXT: v_mul_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v20, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v21, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4 +; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fmul_v32bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v22 -; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10 +; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13 +; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_mul_f32_e32 v39, v48, v39 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17 +; GFX10-NEXT: v_mul_f32_e32 v11, v11, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1 ; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_mul_f32_e32 v5, v5, v21 -; GFX10-NEXT: v_mul_f32_e32 v21, v53, v52 -; GFX10-NEXT: v_mul_f32_e32 v6, v6, v22 -; GFX10-NEXT: v_mul_f32_e32 v22, v55, v54 -; GFX10-NEXT: v_mul_f32_e32 v7, v7, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v25 -; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v9 +; GFX10-NEXT: v_mul_f32_e32 v49, v50, v49 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16 +; GFX10-NEXT: v_mul_f32_e32 v10, v10, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9 ; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX10-NEXT: v_mul_f32_e32 v32, v33, v32 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v10 -; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_mul_f32_e32 v33, v34, v33 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20 +; GFX10-NEXT: v_mul_f32_e32 v14, v14, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4 +; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_mul_f32_e32 v35, v36, v35 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19 +; GFX10-NEXT: v_mul_f32_e32 v13, v13, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3 +; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_mul_f32_e32 v37, v38, v37 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18 +; GFX10-NEXT: v_mul_f32_e32 v12, v12, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_mul_f32_e32 v0, v0, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX10-NEXT: v_mul_f32_e32 v34, v35, v34 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v11 -; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v28 -; GFX10-NEXT: v_mul_f32_e32 v36, v37, v36 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v12 -; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_mul_f32_e32 v51, v52, v51 +; GFX10-NEXT: v_mul_f32_e32 v9, v9, v25 +; GFX10-NEXT: v_mul_f32_e32 v25, v54, v53 +; GFX10-NEXT: v_mul_f32_e32 v8, v8, v24 +; GFX10-NEXT: v_mul_f32_e32 v24, v64, v55 +; GFX10-NEXT: v_mul_f32_e32 v7, v7, v23 +; GFX10-NEXT: v_mul_f32_e32 v23, v66, v65 +; GFX10-NEXT: v_mul_f32_e32 v6, v6, v22 +; GFX10-NEXT: v_mul_f32_e32 v22, v68, v67 +; GFX10-NEXT: v_mul_f32_e32 v5, v5, v21 +; GFX10-NEXT: v_mul_f32_e32 v21, v30, v34 +; GFX10-NEXT: v_mul_f32_e32 v29, v29, v36 +; GFX10-NEXT: v_mul_f32_e32 v28, v28, v38 +; GFX10-NEXT: v_mul_f32_e32 v27, v27, v48 +; GFX10-NEXT: v_mul_f32_e32 v26, v26, v50 ; GFX10-NEXT: v_mul_f32_e32 v2, v2, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v29 -; GFX10-NEXT: v_mul_f32_e32 v38, v39, v38 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v13 -; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX10-NEXT: v_mul_f32_e32 v3, v3, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v30 -; GFX10-NEXT: v_mul_f32_e32 v48, v49, v48 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v14 -; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX10-NEXT: v_mul_f32_e32 v4, v4, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v15 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX10-NEXT: v_mul_f32_e32 v50, v51, v50 -; GFX10-NEXT: v_mul_f32_e32 v23, v65, v64 -; GFX10-NEXT: v_mul_f32_e32 v8, v8, v24 -; GFX10-NEXT: v_mul_f32_e32 v24, v67, v66 -; GFX10-NEXT: v_mul_f32_e32 v9, v9, v25 -; GFX10-NEXT: v_mul_f32_e32 v25, v33, v68 -; GFX10-NEXT: v_mul_f32_e32 v10, v10, v26 -; GFX10-NEXT: v_mul_f32_e32 v16, v35, v16 -; GFX10-NEXT: v_mul_f32_e32 v11, v11, v27 -; GFX10-NEXT: v_mul_f32_e32 v17, v37, v17 -; GFX10-NEXT: v_mul_f32_e32 v12, v12, v28 -; GFX10-NEXT: v_mul_f32_e32 v18, v39, v18 -; GFX10-NEXT: v_mul_f32_e32 v13, v13, v29 -; GFX10-NEXT: v_mul_f32_e32 v19, v49, v19 -; GFX10-NEXT: v_mul_f32_e32 v14, v14, v30 -; GFX10-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v34, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v36, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v38, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v48, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v50, 0x7060302 -; GFX10-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX10-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX10-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX10-NEXT: v_perm_b32 v11, v11, v16, 0x7060302 -; GFX10-NEXT: v_perm_b32 v12, v12, v17, 0x7060302 -; GFX10-NEXT: v_perm_b32 v13, v13, v18, 0x7060302 -; GFX10-NEXT: v_perm_b32 v14, v14, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302 +; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302 +; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302 +; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302 +; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302 +; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v31 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v31 -; GFX10-NEXT: v_mul_f32_e32 v20, v20, v21 -; GFX10-NEXT: v_mul_f32_e32 v15, v15, v22 -; GFX10-NEXT: v_perm_b32 v15, v15, v20, 0x7060302 +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31 +; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX10-NEXT: v_mul_f32_e32 v16, v32, v16 +; GFX10-NEXT: v_mul_f32_e32 v15, v15, v17 +; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fmul_v32bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v10 -; GFX11-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v27 -; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX11-NEXT: v_dual_mul_f32 v10, v10, v26 :: v_dual_and_b32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v25 +; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v17 +; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16 +; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX11-NEXT: v_dual_mul_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4 +; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v28 -; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v22 +; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX11-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24 +; GFX11-NEXT: v_dual_mul_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10 +; GFX11-NEXT: v_dual_mul_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_dual_mul_f32 v2, v2, v18 :: v_dual_mul_f32 v3, v3, v19 +; GFX11-NEXT: v_dual_mul_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26 +; GFX11-NEXT: v_dual_mul_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26 +; GFX11-NEXT: v_mul_f32_e32 v6, v6, v22 +; GFX11-NEXT: v_dual_mul_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28 ; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v29 -; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v13 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_mul_f32_e32 v10, v10, v26 +; GFX11-NEXT: v_mul_f32_e32 v26, v52, v51 +; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX11-NEXT: v_mul_f32_e32 v25, v54, v53 +; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 ; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13 ; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v30 -; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX11-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v15 -; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX11-NEXT: v_dual_mul_f32 v11, v11, v27 :: v_dual_and_b32 v20, 0xffff0000, v20 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_dual_mul_f32 v26, v71, v70 :: v_dual_lshlrev_b32 v49, 16, v4 -; GFX11-NEXT: v_dual_mul_f32 v13, v13, v29 :: v_dual_and_b32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_mul_f32_e32 v4, v4, v20 -; GFX11-NEXT: v_dual_mul_f32 v8, v8, v24 :: v_dual_mul_f32 v9, v9, v25 -; GFX11-NEXT: v_mul_f32_e32 v25, v69, v68 -; GFX11-NEXT: v_dual_mul_f32 v20, v51, v50 :: v_dual_lshlrev_b32 v39, 16, v3 -; GFX11-NEXT: v_mul_f32_e32 v27, v81, v80 -; GFX11-NEXT: v_mul_f32_e32 v12, v12, v28 -; GFX11-NEXT: v_dual_mul_f32 v28, v83, v82 :: v_dual_mul_f32 v29, v85, v84 -; GFX11-NEXT: v_dual_mul_f32 v6, v6, v22 :: v_dual_and_b32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_mul_f32_e32 v22, v55, v54 -; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX11-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_dual_mul_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27 +; GFX11-NEXT: v_mul_f32_e32 v24, v64, v55 +; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX11-NEXT: v_mul_f32_e32 v7, v7, v23 +; GFX11-NEXT: v_mul_f32_e32 v23, v66, v65 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_mul_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_dual_mul_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30 +; GFX11-NEXT: v_dual_mul_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX11-NEXT: v_dual_mul_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_dual_mul_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30 +; GFX11-NEXT: v_mul_f32_e32 v29, v38, v37 +; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_mul_f32_e32 v37, v86, v85 +; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 ; GFX11-NEXT: v_mul_f32_e32 v14, v14, v30 -; GFX11-NEXT: v_dual_mul_f32 v7, v7, v23 :: v_dual_and_b32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_mul_f32_e32 v23, v65, v64 -; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX11-NEXT: v_dual_mul_f32 v24, v67, v66 :: v_dual_and_b32 v21, 0xffff0000, v21 -; GFX11-NEXT: v_mul_f32_e32 v2, v2, v18 -; GFX11-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v32, 16, v16 -; GFX11-NEXT: v_mul_f32_e32 v18, v39, v38 -; GFX11-NEXT: v_dual_mul_f32 v3, v3, v19 :: v_dual_and_b32 v16, 0xffff0000, v16 -; GFX11-NEXT: v_mul_f32_e32 v19, v49, v48 -; GFX11-NEXT: v_mul_f32_e32 v17, v37, v36 -; GFX11-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX11-NEXT: v_dual_mul_f32 v5, v5, v21 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_mul_f32_e32 v21, v53, v52 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v2, v2, v17, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v18, 0x7060302 -; GFX11-NEXT: v_mul_f32_e32 v0, v0, v16 -; GFX11-NEXT: v_mul_f32_e32 v16, v35, v34 -; GFX11-NEXT: v_mul_f32_e32 v32, v33, v32 -; GFX11-NEXT: v_perm_b32 v4, v4, v19, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v20, 0x7060302 -; GFX11-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX11-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX11-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX11-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX11-NEXT: v_perm_b32 v11, v11, v26, 0x7060302 -; GFX11-NEXT: v_perm_b32 v12, v12, v27, 0x7060302 -; GFX11-NEXT: v_perm_b32 v13, v13, v28, 0x7060302 -; GFX11-NEXT: v_perm_b32 v14, v14, v29, 0x7060302 -; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_mul_f32 v16, v86, v16 :: v_dual_and_b32 v17, 0xffff0000, v31 -; GFX11-NEXT: v_mul_f32_e32 v15, v15, v17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_mul_f32 v30, v36, v35 :: v_dual_mul_f32 v33, v34, v33 +; GFX11-NEXT: v_dual_mul_f32 v34, v80, v71 :: v_dual_mul_f32 v35, v82, v81 +; GFX11-NEXT: v_mul_f32_e32 v36, v84, v83 +; GFX11-NEXT: v_dual_mul_f32 v16, v32, v16 :: v_dual_mul_f32 v15, v15, v17 +; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302 +; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302 +; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302 +; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302 +; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302 +; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302 +; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fmul <32 x bfloat> %a, %b @@ -13396,83 +13280,52 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GFX8-LABEL: v_minnum_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_min_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_min_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_min_f32_e32 v3, v4, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_min_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_min_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minnum_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_min_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_min_f32_e32 v3, v4, v3 ; GFX9-NEXT: v_min_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_min_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minnum_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_min_f32_e32 v4, v5, v4 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_min_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_min_f32_e32 v1, v1, v3 ; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_minnum_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_min_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_min_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_min_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_min_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <3 x bfloat> @llvm.minnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) ret <3 x bfloat> %op } @@ -13539,82 +13392,81 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GFX8-LABEL: v_minnum_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_min_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_min_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_min_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_min_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_min_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_min_f32_e32 v3, v5, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minnum_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_min_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_min_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_min_f32_e32 v3, v5, v3 ; GFX9-NEXT: v_min_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_min_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minnum_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_min_f32_e32 v4, v5, v4 +; GFX10-NEXT: v_min_f32_e32 v5, v7, v6 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_min_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_min_f32_e32 v1, v1, v3 -; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minnum_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_min_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_min_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_min_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_min_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_dual_min_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_min_f32_e32 v1, v1, v3 +; GFX11-NEXT: v_min_f32_e32 v5, v7, v6 +; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <4 x bfloat> @llvm.minnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) ret <4 x bfloat> %op @@ -13730,138 +13582,138 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GFX8-LABEL: v_minnum_v8bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX8-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX8-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_min_f32_e32 v7, v9, v7 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_min_f32_e32 v6, v9, v6 ; GFX8-NEXT: v_min_f32_e32 v1, v1, v5 -; GFX8-NEXT: v_min_f32_e32 v4, v8, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX8-NEXT: v_min_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_min_f32_e32 v2, v2, v5 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX8-NEXT: v_min_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_min_f32_e32 v3, v3, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_min_f32_e32 v0, v0, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v4, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_min_f32_e32 v5, v9, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v7, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minnum_v8bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_min_f32_e32 v0, v0, v4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 +; GFX9-NEXT: v_min_f32_e32 v3, v3, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_min_f32_e32 v7, v9, v7 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1 ; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_min_f32_e32 v4, v8, v4 +; GFX9-NEXT: v_min_f32_e32 v6, v9, v6 ; GFX9-NEXT: v_min_f32_e32 v1, v1, v5 -; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX9-NEXT: v_min_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_min_f32_e32 v2, v2, v5 -; GFX9-NEXT: v_perm_b32 v2, v2, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX9-NEXT: v_min_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_min_f32_e32 v3, v3, v5 -; GFX9-NEXT: v_perm_b32 v3, v3, v4, s4 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_min_f32_e32 v5, v9, v5 +; GFX9-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX9-NEXT: s_mov_b32 s4, 0x7060302 +; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v7, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minnum_v8bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_min_f32_e32 v8, v9, v8 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_min_f32_e32 v9, v11, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX10-NEXT: v_min_f32_e32 v8, v9, v8 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_min_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_min_f32_e32 v4, v11, v10 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_min_f32_e32 v10, v11, v10 +; GFX10-NEXT: v_min_f32_e32 v11, v13, v12 +; GFX10-NEXT: v_min_f32_e32 v0, v0, v4 ; GFX10-NEXT: v_min_f32_e32 v1, v1, v5 -; GFX10-NEXT: v_min_f32_e32 v5, v10, v9 ; GFX10-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX10-NEXT: v_min_f32_e32 v6, v12, v11 ; GFX10-NEXT: v_min_f32_e32 v3, v3, v7 -; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minnum_v8bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_min_f32 v9, v11, v10 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_min_f32_e32 v0, v0, v4 ; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_min_f32 v1, v1, v5 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_min_f32_e32 v0, v0, v4 -; GFX11-NEXT: v_min_f32_e32 v4, v11, v10 -; GFX11-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_min_f32_e32 v5, v10, v9 -; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX11-NEXT: v_min_f32_e32 v2, v2, v6 -; GFX11-NEXT: v_min_f32_e32 v6, v12, v11 +; GFX11-NEXT: v_dual_min_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_dual_min_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3 ; GFX11-NEXT: v_min_f32_e32 v3, v3, v7 -; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX11-NEXT: v_dual_min_f32 v10, v11, v10 :: v_dual_min_f32 v11, v13, v12 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <8 x bfloat> @llvm.minnum.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) ret <8 x bfloat> %op @@ -14077,252 +13929,254 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX8-LABEL: v_minnum_v16bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_min_f32_e32 v0, v0, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX8-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v1 +; GFX8-NEXT: v_min_f32_e32 v7, v7, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_min_f32_e32 v15, v17, v15 +; GFX8-NEXT: v_min_f32_e32 v6, v6, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_min_f32_e32 v14, v17, v14 +; GFX8-NEXT: v_min_f32_e32 v5, v5, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_min_f32_e32 v13, v17, v13 +; GFX8-NEXT: v_min_f32_e32 v4, v4, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_min_f32_e32 v12, v17, v12 +; GFX8-NEXT: v_min_f32_e32 v3, v3, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_min_f32_e32 v11, v17, v11 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_min_f32_e32 v10, v17, v10 ; GFX8-NEXT: v_min_f32_e32 v1, v1, v9 -; GFX8-NEXT: v_min_f32_e32 v8, v16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX8-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_min_f32_e32 v2, v2, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX8-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_min_f32_e32 v3, v3, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX8-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_min_f32_e32 v4, v4, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX8-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_min_f32_e32 v5, v5, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX8-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_min_f32_e32 v6, v6, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX8-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_min_f32_e32 v7, v7, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_min_f32_e32 v0, v0, v8 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v8, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_min_f32_e32 v9, v17, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v11, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v12, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v13, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v14, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v15, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minnum_v16bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v8 +; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_min_f32_e32 v7, v7, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_min_f32_e32 v15, v17, v15 +; GFX9-NEXT: v_min_f32_e32 v6, v6, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_min_f32_e32 v14, v17, v14 +; GFX9-NEXT: v_min_f32_e32 v5, v5, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_min_f32_e32 v13, v17, v13 +; GFX9-NEXT: v_min_f32_e32 v4, v4, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_min_f32_e32 v12, v17, v12 +; GFX9-NEXT: v_min_f32_e32 v3, v3, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_min_f32_e32 v11, v17, v11 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_min_f32_e32 v10, v17, v10 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8 ; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_min_f32_e32 v9, v17, v9 ; GFX9-NEXT: v_min_f32_e32 v0, v0, v8 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_min_f32_e32 v8, v16, v8 -; GFX9-NEXT: v_min_f32_e32 v1, v1, v9 -; GFX9-NEXT: v_perm_b32 v1, v1, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_min_f32_e32 v2, v2, v9 -; GFX9-NEXT: v_perm_b32 v2, v2, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_min_f32_e32 v3, v3, v9 -; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_min_f32_e32 v4, v4, v9 -; GFX9-NEXT: v_perm_b32 v4, v4, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_min_f32_e32 v5, v5, v9 -; GFX9-NEXT: v_perm_b32 v5, v5, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_min_f32_e32 v6, v6, v9 -; GFX9-NEXT: v_perm_b32 v6, v6, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX9-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_min_f32_e32 v7, v7, v9 -; GFX9-NEXT: v_perm_b32 v7, v7, v8, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v11, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v12, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v13, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v14, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v15, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minnum_v16bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5 ; GFX10-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX10-NEXT: v_min_f32_e32 v7, v7, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX10-NEXT: v_min_f32_e32 v15, v17, v15 +; GFX10-NEXT: v_min_f32_e32 v6, v6, v14 +; GFX10-NEXT: v_min_f32_e32 v14, v19, v18 +; GFX10-NEXT: v_min_f32_e32 v5, v5, v13 +; GFX10-NEXT: v_min_f32_e32 v13, v21, v20 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_min_f32_e32 v17, v18, v17 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_min_f32_e32 v18, v20, v19 ; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX10-NEXT: v_min_f32_e32 v19, v20, v19 +; GFX10-NEXT: v_min_f32_e32 v20, v22, v21 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v8 -; GFX10-NEXT: v_min_f32_e32 v8, v18, v17 ; GFX10-NEXT: v_min_f32_e32 v1, v1, v9 -; GFX10-NEXT: v_min_f32_e32 v9, v20, v19 ; GFX10-NEXT: v_min_f32_e32 v2, v2, v10 -; GFX10-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v11 -; GFX10-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX10-NEXT: v_min_f32_e32 v8, v9, v8 -; GFX10-NEXT: v_min_f32_e32 v3, v3, v10 -; GFX10-NEXT: v_min_f32_e32 v9, v16, v11 +; GFX10-NEXT: v_min_f32_e32 v3, v3, v11 ; GFX10-NEXT: v_min_f32_e32 v4, v4, v12 -; GFX10-NEXT: v_min_f32_e32 v10, v18, v17 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_min_f32_e32 v5, v5, v11 -; GFX10-NEXT: v_min_f32_e32 v11, v13, v12 -; GFX10-NEXT: v_min_f32_e32 v6, v6, v14 -; GFX10-NEXT: v_min_f32_e32 v12, v17, v16 -; GFX10-NEXT: v_min_f32_e32 v7, v7, v15 -; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX10-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minnum_v16bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_min_f32_e32 v2, v2, v10 -; GFX11-NEXT: v_dual_min_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_min_f32 v1, v1, v9 :: v_dual_and_b32 v10, 0xffff0000, v11 -; GFX11-NEXT: v_min_f32_e32 v9, v20, v19 -; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_min_f32 v3, v3, v10 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_min_f32_e32 v0, v0, v8 -; GFX11-NEXT: v_min_f32_e32 v8, v18, v17 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_dual_min_f32 v10, v18, v17 :: v_dual_lshlrev_b32 v17, 16, v7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_dual_min_f32 v4, v4, v12 :: v_dual_and_b32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_min_f32 v9, v16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_min_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX11-NEXT: v_min_f32_e32 v5, v5, v13 +; GFX11-NEXT: v_min_f32_e32 v13, v21, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_dual_min_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_min_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX11-NEXT: v_min_f32_e32 v7, v7, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX11-NEXT: v_min_f32_e32 v5, v5, v11 -; GFX11-NEXT: v_min_f32_e32 v11, v13, v12 -; GFX11-NEXT: v_min_f32_e32 v12, v17, v16 -; GFX11-NEXT: v_dual_min_f32 v6, v6, v14 :: v_dual_min_f32 v7, v7, v15 -; GFX11-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 +; GFX11-NEXT: v_min_f32_e32 v15, v17, v15 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_dual_min_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11 +; GFX11-NEXT: v_min_f32_e32 v14, v19, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 +; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_dual_min_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX11-NEXT: v_min_f32_e32 v3, v3, v11 +; GFX11-NEXT: v_dual_min_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_dual_min_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_min_f32 v19, v20, v19 :: v_dual_min_f32 v20, v22, v21 +; GFX11-NEXT: v_min_f32_e32 v1, v1, v9 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <16 x bfloat> @llvm.minnum.v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) ret <16 x bfloat> %op @@ -14858,483 +14712,480 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GFX8-LABEL: v_minnum_v32bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_min_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX8-NEXT: v_min_f32_e32 v31, v32, v31 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v31, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_min_f32_e32 v1, v1, v17 -; GFX8-NEXT: v_min_f32_e32 v16, v31, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX8-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_min_f32_e32 v2, v2, v17 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX8-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_min_f32_e32 v3, v3, v17 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX8-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_min_f32_e32 v4, v4, v17 -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v21 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_min_f32_e32 v5, v5, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v6 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v22 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_min_f32_e32 v6, v6, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v7 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v23 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_min_f32_e32 v7, v7, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v24 +; GFX8-NEXT: v_min_f32_e32 v30, v14, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_min_f32_e32 v14, v32, v14 +; GFX8-NEXT: v_min_f32_e32 v13, v13, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_min_f32_e32 v29, v32, v29 +; GFX8-NEXT: v_min_f32_e32 v12, v12, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_min_f32_e32 v28, v32, v28 +; GFX8-NEXT: v_min_f32_e32 v11, v11, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_min_f32_e32 v27, v32, v27 +; GFX8-NEXT: v_min_f32_e32 v10, v10, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX8-NEXT: v_min_f32_e32 v26, v32, v26 +; GFX8-NEXT: v_min_f32_e32 v9, v9, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 ; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_min_f32_e32 v8, v8, v18 +; GFX8-NEXT: v_min_f32_e32 v8, v8, v24 +; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX8-NEXT: v_min_f32_e32 v25, v32, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_alignbit_b32 v8, v8, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v25 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX8-NEXT: v_min_f32_e32 v9, v9, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 -; GFX8-NEXT: v_alignbit_b32 v9, v9, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v10 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v26 -; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX8-NEXT: v_min_f32_e32 v10, v10, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX8-NEXT: v_alignbit_b32 v10, v10, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v11 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v27 -; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX8-NEXT: v_min_f32_e32 v11, v11, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_alignbit_b32 v11, v11, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v12 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v28 -; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX8-NEXT: v_min_f32_e32 v12, v12, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_alignbit_b32 v12, v12, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v29 -; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX8-NEXT: v_min_f32_e32 v13, v13, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_alignbit_b32 v13, v13, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v14 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v30 -; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX8-NEXT: v_min_f32_e32 v14, v14, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_alignbit_b32 v14, v14, v16, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16 +; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16 +; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16 +; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16 +; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16 +; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX8-NEXT: v_min_f32_e32 v32, v32, v33 +; GFX8-NEXT: v_min_f32_e32 v15, v15, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_min_f32_e32 v24, v33, v24 +; GFX8-NEXT: v_min_f32_e32 v7, v7, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_min_f32_e32 v23, v33, v23 +; GFX8-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_min_f32_e32 v22, v33, v22 +; GFX8-NEXT: v_min_f32_e32 v5, v5, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_min_f32_e32 v21, v33, v21 +; GFX8-NEXT: v_min_f32_e32 v4, v4, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_min_f32_e32 v20, v33, v20 +; GFX8-NEXT: v_min_f32_e32 v3, v3, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_min_f32_e32 v19, v33, v19 +; GFX8-NEXT: v_min_f32_e32 v2, v2, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX8-NEXT: v_min_f32_e32 v15, v15, v17 -; GFX8-NEXT: v_min_f32_e32 v16, v18, v16 +; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_min_f32_e32 v18, v33, v18 +; GFX8-NEXT: v_min_f32_e32 v1, v1, v17 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_min_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_min_f32_e32 v17, v33, v17 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_alignbit_b32 v15, v15, v16, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v20, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v21, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16 +; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16 +; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_minnum_v32bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX9-NEXT: v_min_f32_e32 v31, v32, v31 -; GFX9-NEXT: v_min_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_min_f32_e32 v14, v14, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_min_f32_e32 v30, v32, v30 +; GFX9-NEXT: v_min_f32_e32 v13, v13, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_min_f32_e32 v29, v32, v29 +; GFX9-NEXT: v_min_f32_e32 v12, v12, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_min_f32_e32 v28, v32, v28 +; GFX9-NEXT: v_min_f32_e32 v11, v11, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_min_f32_e32 v27, v32, v27 +; GFX9-NEXT: v_min_f32_e32 v10, v10, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_min_f32_e32 v26, v32, v26 +; GFX9-NEXT: v_min_f32_e32 v9, v9, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX9-NEXT: v_min_f32_e32 v8, v8, v24 +; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX9-NEXT: v_min_f32_e32 v25, v32, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v31, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4 +; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4 +; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4 +; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4 +; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4 +; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4 +; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_min_f32_e32 v32, v32, v33 +; GFX9-NEXT: v_min_f32_e32 v15, v15, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_min_f32_e32 v24, v33, v24 +; GFX9-NEXT: v_min_f32_e32 v7, v7, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_min_f32_e32 v23, v33, v23 +; GFX9-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_min_f32_e32 v22, v33, v22 +; GFX9-NEXT: v_min_f32_e32 v5, v5, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_min_f32_e32 v21, v33, v21 +; GFX9-NEXT: v_min_f32_e32 v4, v4, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_min_f32_e32 v20, v33, v20 +; GFX9-NEXT: v_min_f32_e32 v3, v3, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_min_f32_e32 v19, v33, v19 +; GFX9-NEXT: v_min_f32_e32 v2, v2, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_min_f32_e32 v16, v31, v16 +; GFX9-NEXT: v_min_f32_e32 v18, v33, v18 ; GFX9-NEXT: v_min_f32_e32 v1, v1, v17 -; GFX9-NEXT: v_perm_b32 v1, v1, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_min_f32_e32 v2, v2, v17 -; GFX9-NEXT: v_perm_b32 v2, v2, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_min_f32_e32 v3, v3, v17 -; GFX9-NEXT: v_perm_b32 v3, v3, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_min_f32_e32 v4, v4, v17 -; GFX9-NEXT: v_perm_b32 v4, v4, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v21 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_min_f32_e32 v5, v5, v17 -; GFX9-NEXT: v_perm_b32 v5, v5, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v22 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_min_f32_e32 v6, v6, v17 -; GFX9-NEXT: v_perm_b32 v6, v6, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v23 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_min_f32_e32 v7, v7, v17 -; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v8 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v24 -; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX9-NEXT: v_min_f32_e32 v8, v8, v17 -; GFX9-NEXT: v_perm_b32 v8, v8, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v25 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_min_f32_e32 v9, v9, v17 -; GFX9-NEXT: v_perm_b32 v9, v9, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v26 -; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX9-NEXT: v_min_f32_e32 v10, v10, v17 -; GFX9-NEXT: v_perm_b32 v10, v10, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v27 -; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX9-NEXT: v_min_f32_e32 v11, v11, v17 -; GFX9-NEXT: v_perm_b32 v11, v11, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v28 -; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX9-NEXT: v_min_f32_e32 v12, v12, v17 -; GFX9-NEXT: v_perm_b32 v12, v12, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v29 -; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX9-NEXT: v_min_f32_e32 v13, v13, v17 -; GFX9-NEXT: v_perm_b32 v13, v13, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v14 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v30 -; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX9-NEXT: v_min_f32_e32 v14, v14, v17 -; GFX9-NEXT: v_perm_b32 v14, v14, v16, s4 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v15 -; GFX9-NEXT: v_min_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX9-NEXT: v_min_f32_e32 v15, v15, v17 -; GFX9-NEXT: v_perm_b32 v15, v15, v16, s4 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_min_f32_e32 v17, v33, v17 +; GFX9-NEXT: v_min_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v20, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v21, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4 +; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_minnum_v32bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v22 -; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10 +; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13 +; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_min_f32_e32 v39, v48, v39 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17 +; GFX10-NEXT: v_min_f32_e32 v11, v11, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1 ; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_min_f32_e32 v5, v5, v21 -; GFX10-NEXT: v_min_f32_e32 v21, v53, v52 -; GFX10-NEXT: v_min_f32_e32 v6, v6, v22 -; GFX10-NEXT: v_min_f32_e32 v22, v55, v54 -; GFX10-NEXT: v_min_f32_e32 v7, v7, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v25 -; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v9 +; GFX10-NEXT: v_min_f32_e32 v49, v50, v49 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16 +; GFX10-NEXT: v_min_f32_e32 v10, v10, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9 ; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX10-NEXT: v_min_f32_e32 v32, v33, v32 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v10 -; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_min_f32_e32 v33, v34, v33 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20 +; GFX10-NEXT: v_min_f32_e32 v14, v14, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4 +; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_min_f32_e32 v35, v36, v35 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19 +; GFX10-NEXT: v_min_f32_e32 v13, v13, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3 +; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_min_f32_e32 v37, v38, v37 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18 +; GFX10-NEXT: v_min_f32_e32 v12, v12, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_min_f32_e32 v0, v0, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX10-NEXT: v_min_f32_e32 v34, v35, v34 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v11 -; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX10-NEXT: v_min_f32_e32 v1, v1, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v28 -; GFX10-NEXT: v_min_f32_e32 v36, v37, v36 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v12 -; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_min_f32_e32 v51, v52, v51 +; GFX10-NEXT: v_min_f32_e32 v9, v9, v25 +; GFX10-NEXT: v_min_f32_e32 v25, v54, v53 +; GFX10-NEXT: v_min_f32_e32 v8, v8, v24 +; GFX10-NEXT: v_min_f32_e32 v24, v64, v55 +; GFX10-NEXT: v_min_f32_e32 v7, v7, v23 +; GFX10-NEXT: v_min_f32_e32 v23, v66, v65 +; GFX10-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX10-NEXT: v_min_f32_e32 v22, v68, v67 +; GFX10-NEXT: v_min_f32_e32 v5, v5, v21 +; GFX10-NEXT: v_min_f32_e32 v21, v30, v34 +; GFX10-NEXT: v_min_f32_e32 v29, v29, v36 +; GFX10-NEXT: v_min_f32_e32 v28, v28, v38 +; GFX10-NEXT: v_min_f32_e32 v27, v27, v48 +; GFX10-NEXT: v_min_f32_e32 v26, v26, v50 ; GFX10-NEXT: v_min_f32_e32 v2, v2, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v29 -; GFX10-NEXT: v_min_f32_e32 v38, v39, v38 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v13 -; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX10-NEXT: v_min_f32_e32 v3, v3, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v30 -; GFX10-NEXT: v_min_f32_e32 v48, v49, v48 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v14 -; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX10-NEXT: v_min_f32_e32 v4, v4, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v15 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX10-NEXT: v_min_f32_e32 v50, v51, v50 -; GFX10-NEXT: v_min_f32_e32 v23, v65, v64 -; GFX10-NEXT: v_min_f32_e32 v8, v8, v24 -; GFX10-NEXT: v_min_f32_e32 v24, v67, v66 -; GFX10-NEXT: v_min_f32_e32 v9, v9, v25 -; GFX10-NEXT: v_min_f32_e32 v25, v33, v68 -; GFX10-NEXT: v_min_f32_e32 v10, v10, v26 -; GFX10-NEXT: v_min_f32_e32 v16, v35, v16 -; GFX10-NEXT: v_min_f32_e32 v11, v11, v27 -; GFX10-NEXT: v_min_f32_e32 v17, v37, v17 -; GFX10-NEXT: v_min_f32_e32 v12, v12, v28 -; GFX10-NEXT: v_min_f32_e32 v18, v39, v18 -; GFX10-NEXT: v_min_f32_e32 v13, v13, v29 -; GFX10-NEXT: v_min_f32_e32 v19, v49, v19 -; GFX10-NEXT: v_min_f32_e32 v14, v14, v30 -; GFX10-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v34, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v36, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v38, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v48, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v50, 0x7060302 -; GFX10-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX10-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX10-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX10-NEXT: v_perm_b32 v11, v11, v16, 0x7060302 -; GFX10-NEXT: v_perm_b32 v12, v12, v17, 0x7060302 -; GFX10-NEXT: v_perm_b32 v13, v13, v18, 0x7060302 -; GFX10-NEXT: v_perm_b32 v14, v14, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302 +; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302 +; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302 +; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302 +; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302 +; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v31 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v31 -; GFX10-NEXT: v_min_f32_e32 v20, v20, v21 -; GFX10-NEXT: v_min_f32_e32 v15, v15, v22 -; GFX10-NEXT: v_perm_b32 v15, v15, v20, 0x7060302 +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31 +; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX10-NEXT: v_min_f32_e32 v16, v32, v16 +; GFX10-NEXT: v_min_f32_e32 v15, v15, v17 +; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_minnum_v32bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v10 -; GFX11-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v27 -; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX11-NEXT: v_dual_min_f32 v10, v10, v26 :: v_dual_and_b32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v25 +; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v17 +; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16 +; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX11-NEXT: v_dual_min_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4 +; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v28 -; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v22 +; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX11-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24 +; GFX11-NEXT: v_dual_min_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10 +; GFX11-NEXT: v_dual_min_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_dual_min_f32 v2, v2, v18 :: v_dual_min_f32 v3, v3, v19 +; GFX11-NEXT: v_dual_min_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26 +; GFX11-NEXT: v_dual_min_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26 +; GFX11-NEXT: v_min_f32_e32 v6, v6, v22 +; GFX11-NEXT: v_dual_min_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28 ; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v29 -; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v13 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_min_f32_e32 v10, v10, v26 +; GFX11-NEXT: v_min_f32_e32 v26, v52, v51 +; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX11-NEXT: v_min_f32_e32 v25, v54, v53 +; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 ; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13 ; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v30 -; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX11-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v15 -; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX11-NEXT: v_dual_min_f32 v11, v11, v27 :: v_dual_and_b32 v20, 0xffff0000, v20 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_dual_min_f32 v26, v71, v70 :: v_dual_lshlrev_b32 v49, 16, v4 -; GFX11-NEXT: v_dual_min_f32 v13, v13, v29 :: v_dual_and_b32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_min_f32_e32 v4, v4, v20 -; GFX11-NEXT: v_dual_min_f32 v8, v8, v24 :: v_dual_min_f32 v9, v9, v25 -; GFX11-NEXT: v_min_f32_e32 v25, v69, v68 -; GFX11-NEXT: v_dual_min_f32 v20, v51, v50 :: v_dual_lshlrev_b32 v39, 16, v3 -; GFX11-NEXT: v_min_f32_e32 v27, v81, v80 -; GFX11-NEXT: v_min_f32_e32 v12, v12, v28 -; GFX11-NEXT: v_dual_min_f32 v28, v83, v82 :: v_dual_min_f32 v29, v85, v84 -; GFX11-NEXT: v_dual_min_f32 v6, v6, v22 :: v_dual_and_b32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_min_f32_e32 v22, v55, v54 -; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX11-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_dual_min_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27 +; GFX11-NEXT: v_min_f32_e32 v24, v64, v55 +; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX11-NEXT: v_min_f32_e32 v7, v7, v23 +; GFX11-NEXT: v_min_f32_e32 v23, v66, v65 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_min_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_dual_min_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30 +; GFX11-NEXT: v_dual_min_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX11-NEXT: v_dual_min_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_dual_min_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30 +; GFX11-NEXT: v_min_f32_e32 v29, v38, v37 +; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_min_f32_e32 v37, v86, v85 +; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 ; GFX11-NEXT: v_min_f32_e32 v14, v14, v30 -; GFX11-NEXT: v_dual_min_f32 v7, v7, v23 :: v_dual_and_b32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_min_f32_e32 v23, v65, v64 -; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX11-NEXT: v_dual_min_f32 v24, v67, v66 :: v_dual_and_b32 v21, 0xffff0000, v21 -; GFX11-NEXT: v_min_f32_e32 v2, v2, v18 -; GFX11-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v32, 16, v16 -; GFX11-NEXT: v_min_f32_e32 v18, v39, v38 -; GFX11-NEXT: v_dual_min_f32 v3, v3, v19 :: v_dual_and_b32 v16, 0xffff0000, v16 -; GFX11-NEXT: v_min_f32_e32 v19, v49, v48 -; GFX11-NEXT: v_min_f32_e32 v17, v37, v36 -; GFX11-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX11-NEXT: v_dual_min_f32 v5, v5, v21 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_min_f32_e32 v21, v53, v52 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v2, v2, v17, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v18, 0x7060302 -; GFX11-NEXT: v_min_f32_e32 v0, v0, v16 -; GFX11-NEXT: v_min_f32_e32 v16, v35, v34 -; GFX11-NEXT: v_min_f32_e32 v32, v33, v32 -; GFX11-NEXT: v_perm_b32 v4, v4, v19, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v20, 0x7060302 -; GFX11-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX11-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX11-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX11-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX11-NEXT: v_perm_b32 v11, v11, v26, 0x7060302 -; GFX11-NEXT: v_perm_b32 v12, v12, v27, 0x7060302 -; GFX11-NEXT: v_perm_b32 v13, v13, v28, 0x7060302 -; GFX11-NEXT: v_perm_b32 v14, v14, v29, 0x7060302 -; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_min_f32 v16, v86, v16 :: v_dual_and_b32 v17, 0xffff0000, v31 -; GFX11-NEXT: v_min_f32_e32 v15, v15, v17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_min_f32 v30, v36, v35 :: v_dual_min_f32 v33, v34, v33 +; GFX11-NEXT: v_dual_min_f32 v34, v80, v71 :: v_dual_min_f32 v35, v82, v81 +; GFX11-NEXT: v_min_f32_e32 v36, v84, v83 +; GFX11-NEXT: v_dual_min_f32 v16, v32, v16 :: v_dual_min_f32 v15, v15, v17 +; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302 +; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302 +; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302 +; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302 +; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302 +; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302 +; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <32 x bfloat> @llvm.minnum.v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) @@ -15553,83 +15404,52 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) { ; GFX8-LABEL: v_maxnum_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_max_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_max_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_max_f32_e32 v4, v5, v4 +; GFX8-NEXT: v_max_f32_e32 v3, v4, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_max_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_max_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maxnum_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_max_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_max_f32_e32 v3, v4, v3 ; GFX9-NEXT: v_max_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_max_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maxnum_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 ; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_max_f32_e32 v4, v5, v4 ; GFX10-NEXT: v_max_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_max_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_max_f32_e32 v1, v1, v3 ; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_maxnum_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_max_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_max_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_max_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_max_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) ret <3 x bfloat> %op } @@ -15696,82 +15516,81 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) { ; GFX8-LABEL: v_maxnum_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_max_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_max_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_max_f32_e32 v4, v5, v4 ; GFX8-NEXT: v_max_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_max_f32_e32 v2, v4, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_max_f32_e32 v0, v0, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_max_f32_e32 v3, v5, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maxnum_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_max_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 ; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_max_f32_e32 v4, v5, v4 +; GFX9-NEXT: v_max_f32_e32 v3, v5, v3 ; GFX9-NEXT: v_max_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_max_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maxnum_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_max_f32_e32 v4, v5, v4 +; GFX10-NEXT: v_max_f32_e32 v5, v7, v6 ; GFX10-NEXT: v_max_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_max_f32_e32 v2, v7, v6 ; GFX10-NEXT: v_max_f32_e32 v1, v1, v3 -; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maxnum_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_max_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_max_f32_e32 v0, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_max_f32_e32 v4, v5, v4 -; GFX11-NEXT: v_max_f32_e32 v2, v7, v6 -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_dual_max_f32 v4, v5, v4 :: v_dual_and_b32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_max_f32_e32 v1, v1, v3 +; GFX11-NEXT: v_max_f32_e32 v5, v7, v6 +; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v5, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <4 x bfloat> @llvm.maxnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) ret <4 x bfloat> %op @@ -15887,138 +15706,138 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) { ; GFX8-LABEL: v_maxnum_v8bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX8-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX8-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_max_f32_e32 v7, v9, v7 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_max_f32_e32 v6, v9, v6 ; GFX8-NEXT: v_max_f32_e32 v1, v1, v5 -; GFX8-NEXT: v_max_f32_e32 v4, v8, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX8-NEXT: v_max_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_max_f32_e32 v2, v2, v5 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v4, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX8-NEXT: v_max_f32_e32 v4, v5, v4 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_max_f32_e32 v3, v3, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_max_f32_e32 v0, v0, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v4, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_max_f32_e32 v5, v9, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v5, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v7, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maxnum_v8bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v4 +; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 +; GFX9-NEXT: v_max_f32_e32 v3, v3, v7 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_max_f32_e32 v7, v9, v7 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_max_f32_e32 v6, v9, v6 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4 ; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 +; GFX9-NEXT: v_max_f32_e32 v5, v9, v5 ; GFX9-NEXT: v_max_f32_e32 v0, v0, v4 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_max_f32_e32 v4, v8, v4 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v5 -; GFX9-NEXT: v_perm_b32 v1, v1, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v6 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2 -; GFX9-NEXT: v_max_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v6 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_max_f32_e32 v2, v2, v5 -; GFX9-NEXT: v_perm_b32 v2, v2, v4, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v7 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v3 -; GFX9-NEXT: v_max_f32_e32 v4, v5, v4 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v7 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_max_f32_e32 v3, v3, v5 -; GFX9-NEXT: v_perm_b32 v3, v3, v4, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v5, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v7, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maxnum_v8bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_max_f32_e32 v8, v9, v8 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_max_f32_e32 v9, v11, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX10-NEXT: v_max_f32_e32 v8, v9, v8 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_max_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_max_f32_e32 v4, v11, v10 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_max_f32_e32 v10, v11, v10 +; GFX10-NEXT: v_max_f32_e32 v11, v13, v12 +; GFX10-NEXT: v_max_f32_e32 v0, v0, v4 ; GFX10-NEXT: v_max_f32_e32 v1, v1, v5 -; GFX10-NEXT: v_max_f32_e32 v5, v10, v9 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX10-NEXT: v_max_f32_e32 v6, v12, v11 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v7 -; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maxnum_v8bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_max_f32 v9, v11, v10 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_max_f32_e32 v0, v0, v4 ; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v4 -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_max_f32 v1, v1, v5 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_max_f32_e32 v0, v0, v4 -; GFX11-NEXT: v_max_f32_e32 v4, v11, v10 -; GFX11-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v9, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v7 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_max_f32_e32 v5, v10, v9 -; GFX11-NEXT: v_perm_b32 v0, v0, v8, 0x7060302 -; GFX11-NEXT: v_max_f32_e32 v2, v2, v6 -; GFX11-NEXT: v_max_f32_e32 v6, v12, v11 +; GFX11-NEXT: v_dual_max_f32 v1, v1, v5 :: v_dual_and_b32 v6, 0xffff0000, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_dual_max_f32 v2, v2, v6 :: v_dual_and_b32 v3, 0xffff0000, v3 ; GFX11-NEXT: v_max_f32_e32 v3, v3, v7 -; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_perm_b32 v2, v2, v5, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v6, 0x7060302 +; GFX11-NEXT: v_dual_max_f32 v10, v11, v10 :: v_dual_max_f32 v11, v13, v12 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v1, v1, v10, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v11, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <8 x bfloat> @llvm.maxnum.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) ret <8 x bfloat> %op @@ -16234,252 +16053,254 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) { ; GFX8-LABEL: v_maxnum_v16bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_max_f32_e32 v0, v0, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 ; GFX8-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v1 +; GFX8-NEXT: v_max_f32_e32 v7, v7, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_max_f32_e32 v15, v17, v15 +; GFX8-NEXT: v_max_f32_e32 v6, v6, v14 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_max_f32_e32 v14, v17, v14 +; GFX8-NEXT: v_max_f32_e32 v5, v5, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_max_f32_e32 v13, v17, v13 +; GFX8-NEXT: v_max_f32_e32 v4, v4, v12 +; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_max_f32_e32 v12, v17, v12 +; GFX8-NEXT: v_max_f32_e32 v3, v3, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_max_f32_e32 v11, v17, v11 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v10 +; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_max_f32_e32 v10, v17, v10 ; GFX8-NEXT: v_max_f32_e32 v1, v1, v9 -; GFX8-NEXT: v_max_f32_e32 v8, v16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX8-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_max_f32_e32 v2, v2, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX8-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_max_f32_e32 v3, v3, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX8-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_max_f32_e32 v4, v4, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX8-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_max_f32_e32 v5, v5, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX8-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_max_f32_e32 v6, v6, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v8, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX8-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_max_f32_e32 v7, v7, v9 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v8 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_max_f32_e32 v0, v0, v8 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v8, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_max_f32_e32 v9, v17, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v10, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v11, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v12, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v13, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v14, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v15, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maxnum_v16bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v8 +; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_max_f32_e32 v7, v7, v15 +; GFX9-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_max_f32_e32 v15, v17, v15 +; GFX9-NEXT: v_max_f32_e32 v6, v6, v14 +; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_max_f32_e32 v14, v17, v14 +; GFX9-NEXT: v_max_f32_e32 v5, v5, v13 +; GFX9-NEXT: v_lshlrev_b32_e32 v13, 16, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_max_f32_e32 v13, v17, v13 +; GFX9-NEXT: v_max_f32_e32 v4, v4, v12 +; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_max_f32_e32 v12, v17, v12 +; GFX9-NEXT: v_max_f32_e32 v3, v3, v11 +; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_max_f32_e32 v11, v17, v11 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v10 +; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_max_f32_e32 v10, v17, v10 +; GFX9-NEXT: v_max_f32_e32 v1, v1, v9 +; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v8 ; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 +; GFX9-NEXT: v_max_f32_e32 v9, v17, v9 ; GFX9-NEXT: v_max_f32_e32 v0, v0, v8 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v9 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_max_f32_e32 v8, v16, v8 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v9 -; GFX9-NEXT: v_perm_b32 v1, v1, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v10 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v2 -; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v10 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_max_f32_e32 v2, v2, v9 -; GFX9-NEXT: v_perm_b32 v2, v2, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v11 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_max_f32_e32 v3, v3, v9 -; GFX9-NEXT: v_perm_b32 v3, v3, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v12 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v4 -; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v12 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_max_f32_e32 v4, v4, v9 -; GFX9-NEXT: v_perm_b32 v4, v4, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v13 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v13 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_max_f32_e32 v5, v5, v9 -; GFX9-NEXT: v_perm_b32 v5, v5, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v14 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v6 -; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v14 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_max_f32_e32 v6, v6, v9 -; GFX9-NEXT: v_perm_b32 v6, v6, v8, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v15 -; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v7 -; GFX9-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v15 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_max_f32_e32 v7, v7, v9 -; GFX9-NEXT: v_perm_b32 v7, v7, v8, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v9, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v10, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v11, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v12, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v13, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v14, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v15, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maxnum_v16bf16: -; GFX10: ; %bb.0: -; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10: ; %bb.0: +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v5 ; GFX10-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX10-NEXT: v_max_f32_e32 v7, v7, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX10-NEXT: v_max_f32_e32 v15, v17, v15 +; GFX10-NEXT: v_max_f32_e32 v6, v6, v14 +; GFX10-NEXT: v_max_f32_e32 v14, v19, v18 +; GFX10-NEXT: v_max_f32_e32 v5, v5, v13 +; GFX10-NEXT: v_max_f32_e32 v13, v21, v20 +; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v10 ; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_max_f32_e32 v17, v18, v17 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_max_f32_e32 v18, v20, v19 ; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX10-NEXT: v_max_f32_e32 v19, v20, v19 +; GFX10-NEXT: v_max_f32_e32 v20, v22, v21 ; GFX10-NEXT: v_max_f32_e32 v0, v0, v8 -; GFX10-NEXT: v_max_f32_e32 v8, v18, v17 ; GFX10-NEXT: v_max_f32_e32 v1, v1, v9 -; GFX10-NEXT: v_max_f32_e32 v9, v20, v19 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v10 -; GFX10-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v11 -; GFX10-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX10-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX10-NEXT: v_max_f32_e32 v8, v9, v8 -; GFX10-NEXT: v_max_f32_e32 v3, v3, v10 -; GFX10-NEXT: v_max_f32_e32 v9, v16, v11 +; GFX10-NEXT: v_max_f32_e32 v3, v3, v11 ; GFX10-NEXT: v_max_f32_e32 v4, v4, v12 -; GFX10-NEXT: v_max_f32_e32 v10, v18, v17 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX10-NEXT: v_lshlrev_b32_e32 v13, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_max_f32_e32 v5, v5, v11 -; GFX10-NEXT: v_max_f32_e32 v11, v13, v12 -; GFX10-NEXT: v_max_f32_e32 v6, v6, v14 -; GFX10-NEXT: v_max_f32_e32 v12, v17, v16 -; GFX10-NEXT: v_max_f32_e32 v7, v7, v15 -; GFX10-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX10-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maxnum_v16bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v0 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_max_f32_e32 v2, v2, v10 -; GFX11-NEXT: v_dual_max_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v17, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_max_f32 v1, v1, v9 :: v_dual_and_b32 v10, 0xffff0000, v11 -; GFX11-NEXT: v_max_f32_e32 v9, v20, v19 -; GFX11-NEXT: v_perm_b32 v2, v2, v9, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_max_f32 v3, v3, v10 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_max_f32_e32 v0, v0, v8 -; GFX11-NEXT: v_max_f32_e32 v8, v18, v17 -; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_perm_b32 v0, v0, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v8, 0x7060302 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v13 +; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_dual_max_f32 v10, v18, v17 :: v_dual_lshlrev_b32 v17, 16, v7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_dual_max_f32 v4, v4, v12 :: v_dual_and_b32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_max_f32 v9, v16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v22, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_max_f32 v4, v4, v12 :: v_dual_and_b32 v5, 0xffff0000, v5 ; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v15 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v13, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v7 +; GFX11-NEXT: v_max_f32_e32 v5, v5, v13 +; GFX11-NEXT: v_max_f32_e32 v13, v21, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v21, 16, v8 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_dual_max_f32 v16, v17, v16 :: v_dual_and_b32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v17, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_max_f32 v0, v0, v8 :: v_dual_and_b32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_perm_b32 v4, v4, v13, 0x7060302 +; GFX11-NEXT: v_max_f32_e32 v7, v7, v15 +; GFX11-NEXT: v_lshlrev_b32_e32 v15, 16, v14 +; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_perm_b32 v3, v3, v8, 0x7060302 -; GFX11-NEXT: v_max_f32_e32 v5, v5, v11 -; GFX11-NEXT: v_max_f32_e32 v11, v13, v12 -; GFX11-NEXT: v_max_f32_e32 v12, v17, v16 -; GFX11-NEXT: v_dual_max_f32 v6, v6, v14 :: v_dual_max_f32 v7, v7, v15 -; GFX11-NEXT: v_perm_b32 v4, v4, v9, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v10, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v6, v6, v11, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v12, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v7, v7, v16, 0x7060302 +; GFX11-NEXT: v_max_f32_e32 v15, v17, v15 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_dual_max_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v17, 16, v11 +; GFX11-NEXT: v_max_f32_e32 v14, v19, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v18, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v19, 16, v10 +; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_dual_max_f32 v17, v18, v17 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_perm_b32 v5, v5, v14, 0x7060302 +; GFX11-NEXT: v_perm_b32 v6, v6, v15, 0x7060302 +; GFX11-NEXT: v_max_f32_e32 v3, v3, v11 +; GFX11-NEXT: v_dual_max_f32 v18, v20, v19 :: v_dual_lshlrev_b32 v19, 16, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v20, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_dual_max_f32 v2, v2, v10 :: v_dual_and_b32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_perm_b32 v3, v3, v17, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_max_f32 v19, v20, v19 :: v_dual_max_f32 v20, v22, v21 +; GFX11-NEXT: v_max_f32_e32 v1, v1, v9 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v2, v2, v18, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v20, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_perm_b32 v1, v1, v19, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <16 x bfloat> @llvm.maxnum.v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) ret <16 x bfloat> %op @@ -17015,483 +16836,480 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) { ; GFX8-LABEL: v_maxnum_v32bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_max_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX8-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX8-NEXT: v_max_f32_e32 v31, v32, v31 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v31, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX8-NEXT: v_max_f32_e32 v30, v14, v30 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX8-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX8-NEXT: v_max_f32_e32 v14, v32, v14 +; GFX8-NEXT: v_max_f32_e32 v13, v13, v29 +; GFX8-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX8-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX8-NEXT: v_max_f32_e32 v29, v32, v29 +; GFX8-NEXT: v_max_f32_e32 v12, v12, v28 +; GFX8-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX8-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX8-NEXT: v_max_f32_e32 v28, v32, v28 +; GFX8-NEXT: v_max_f32_e32 v11, v11, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX8-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX8-NEXT: v_max_f32_e32 v27, v32, v27 +; GFX8-NEXT: v_max_f32_e32 v10, v10, v26 +; GFX8-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX8-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX8-NEXT: v_max_f32_e32 v26, v32, v26 +; GFX8-NEXT: v_max_f32_e32 v9, v9, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX8-NEXT: v_max_f32_e32 v8, v8, v24 +; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX8-NEXT: v_max_f32_e32 v25, v32, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 +; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 +; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX8-NEXT: v_alignbit_b32 v8, v8, v25, 16 +; GFX8-NEXT: v_alignbit_b32 v9, v9, v26, 16 +; GFX8-NEXT: v_alignbit_b32 v10, v10, v27, 16 +; GFX8-NEXT: v_alignbit_b32 v11, v11, v28, 16 +; GFX8-NEXT: v_alignbit_b32 v12, v12, v29, 16 +; GFX8-NEXT: v_alignbit_b32 v13, v13, v14, 16 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX8-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX8-NEXT: v_max_f32_e32 v32, v32, v33 +; GFX8-NEXT: v_max_f32_e32 v15, v15, v24 +; GFX8-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX8-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX8-NEXT: v_max_f32_e32 v24, v33, v24 +; GFX8-NEXT: v_max_f32_e32 v7, v7, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX8-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_max_f32_e32 v23, v33, v23 +; GFX8-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX8-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX8-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX8-NEXT: v_max_f32_e32 v22, v33, v22 +; GFX8-NEXT: v_max_f32_e32 v5, v5, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX8-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_max_f32_e32 v21, v33, v21 +; GFX8-NEXT: v_max_f32_e32 v4, v4, v20 +; GFX8-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX8-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_max_f32_e32 v20, v33, v20 +; GFX8-NEXT: v_max_f32_e32 v3, v3, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_max_f32_e32 v19, v33, v19 +; GFX8-NEXT: v_max_f32_e32 v2, v2, v18 +; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_max_f32_e32 v18, v33, v18 ; GFX8-NEXT: v_max_f32_e32 v1, v1, v17 -; GFX8-NEXT: v_max_f32_e32 v16, v31, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX8-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_max_f32_e32 v0, v0, v16 +; GFX8-NEXT: v_max_f32_e32 v17, v33, v17 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX8-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_max_f32_e32 v2, v2, v17 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v2, v2, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX8-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_max_f32_e32 v3, v3, v17 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v3, v3, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX8-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX8-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_max_f32_e32 v4, v4, v17 -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_alignbit_b32 v4, v4, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v5 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v21 -; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX8-NEXT: v_max_f32_e32 v5, v5, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_alignbit_b32 v5, v5, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v6 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v22 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_max_f32_e32 v6, v6, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_alignbit_b32 v6, v6, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v7 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v23 -; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX8-NEXT: v_max_f32_e32 v7, v7, v18 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_alignbit_b32 v7, v7, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v8 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v24 -; GFX8-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX8-NEXT: v_max_f32_e32 v8, v8, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_alignbit_b32 v8, v8, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v9 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v25 -; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX8-NEXT: v_max_f32_e32 v9, v9, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 -; GFX8-NEXT: v_alignbit_b32 v9, v9, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v10 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v26 -; GFX8-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX8-NEXT: v_max_f32_e32 v10, v10, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX8-NEXT: v_alignbit_b32 v10, v10, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v11 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v27 -; GFX8-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX8-NEXT: v_max_f32_e32 v11, v11, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_alignbit_b32 v11, v11, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v12 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v28 -; GFX8-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX8-NEXT: v_max_f32_e32 v12, v12, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_alignbit_b32 v12, v12, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v13 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v29 -; GFX8-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX8-NEXT: v_max_f32_e32 v13, v13, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_alignbit_b32 v13, v13, v16, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v14 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 -; GFX8-NEXT: v_and_b32_e32 v18, 0xffff0000, v30 -; GFX8-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX8-NEXT: v_max_f32_e32 v14, v14, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_alignbit_b32 v14, v14, v16, 16 -; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX8-NEXT: v_lshlrev_b32_e32 v18, 16, v15 -; GFX8-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX8-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX8-NEXT: v_max_f32_e32 v15, v15, v17 -; GFX8-NEXT: v_max_f32_e32 v16, v18, v16 ; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_alignbit_b32 v15, v15, v16, 16 +; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v30 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v17, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v18, 16 +; GFX8-NEXT: v_alignbit_b32 v2, v2, v19, 16 +; GFX8-NEXT: v_alignbit_b32 v3, v3, v20, 16 +; GFX8-NEXT: v_alignbit_b32 v4, v4, v21, 16 +; GFX8-NEXT: v_alignbit_b32 v5, v5, v22, 16 +; GFX8-NEXT: v_alignbit_b32 v6, v6, v23, 16 +; GFX8-NEXT: v_alignbit_b32 v7, v7, v24, 16 +; GFX8-NEXT: v_alignbit_b32 v14, v16, v31, 16 +; GFX8-NEXT: v_alignbit_b32 v15, v15, v32, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_maxnum_v32bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v16 -; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v0 -; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v14 +; GFX9-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX9-NEXT: v_max_f32_e32 v31, v32, v31 -; GFX9-NEXT: v_max_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_max_f32_e32 v14, v14, v30 +; GFX9-NEXT: v_lshlrev_b32_e32 v30, 16, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v13 +; GFX9-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX9-NEXT: v_max_f32_e32 v30, v32, v30 +; GFX9-NEXT: v_max_f32_e32 v13, v13, v29 +; GFX9-NEXT: v_lshlrev_b32_e32 v29, 16, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v12 +; GFX9-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX9-NEXT: v_max_f32_e32 v29, v32, v29 +; GFX9-NEXT: v_max_f32_e32 v12, v12, v28 +; GFX9-NEXT: v_lshlrev_b32_e32 v28, 16, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v11 +; GFX9-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX9-NEXT: v_max_f32_e32 v28, v32, v28 +; GFX9-NEXT: v_max_f32_e32 v11, v11, v27 +; GFX9-NEXT: v_lshlrev_b32_e32 v27, 16, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v10 +; GFX9-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX9-NEXT: v_max_f32_e32 v27, v32, v27 +; GFX9-NEXT: v_max_f32_e32 v10, v10, v26 +; GFX9-NEXT: v_lshlrev_b32_e32 v26, 16, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v9 +; GFX9-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 +; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX9-NEXT: v_max_f32_e32 v26, v32, v26 +; GFX9-NEXT: v_max_f32_e32 v9, v9, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v25, 16, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v8 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX9-NEXT: v_max_f32_e32 v8, v8, v24 +; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 +; GFX9-NEXT: v_max_f32_e32 v25, v32, v25 +; GFX9-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v31, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v17 -; GFX9-NEXT: v_lshlrev_b32_e32 v31, 16, v1 +; GFX9-NEXT: v_perm_b32 v8, v8, v25, s4 +; GFX9-NEXT: v_perm_b32 v9, v9, v26, s4 +; GFX9-NEXT: v_perm_b32 v10, v10, v27, s4 +; GFX9-NEXT: v_perm_b32 v11, v11, v28, s4 +; GFX9-NEXT: v_perm_b32 v12, v12, v29, s4 +; GFX9-NEXT: v_perm_b32 v13, v13, v30, s4 +; GFX9-NEXT: v_perm_b32 v14, v14, v31, s4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v24 +; GFX9-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX9-NEXT: v_max_f32_e32 v32, v32, v33 +; GFX9-NEXT: v_max_f32_e32 v15, v15, v24 +; GFX9-NEXT: v_lshlrev_b32_e32 v24, 16, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v7 +; GFX9-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_max_f32_e32 v24, v33, v24 +; GFX9-NEXT: v_max_f32_e32 v7, v7, v23 +; GFX9-NEXT: v_lshlrev_b32_e32 v23, 16, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v6 +; GFX9-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_max_f32_e32 v23, v33, v23 +; GFX9-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX9-NEXT: v_lshlrev_b32_e32 v22, 16, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v5 +; GFX9-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_max_f32_e32 v22, v33, v22 +; GFX9-NEXT: v_max_f32_e32 v5, v5, v21 +; GFX9-NEXT: v_lshlrev_b32_e32 v21, 16, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_max_f32_e32 v21, v33, v21 +; GFX9-NEXT: v_max_f32_e32 v4, v4, v20 +; GFX9-NEXT: v_lshlrev_b32_e32 v20, 16, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v3 +; GFX9-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_max_f32_e32 v20, v33, v20 +; GFX9-NEXT: v_max_f32_e32 v3, v3, v19 +; GFX9-NEXT: v_lshlrev_b32_e32 v19, 16, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v2 +; GFX9-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_max_f32_e32 v19, v33, v19 +; GFX9-NEXT: v_max_f32_e32 v2, v2, v18 +; GFX9-NEXT: v_lshlrev_b32_e32 v18, 16, v17 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v1 ; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_max_f32_e32 v16, v31, v16 +; GFX9-NEXT: v_max_f32_e32 v18, v33, v18 ; GFX9-NEXT: v_max_f32_e32 v1, v1, v17 -; GFX9-NEXT: v_perm_b32 v1, v1, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v2 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_max_f32_e32 v2, v2, v17 -; GFX9-NEXT: v_perm_b32 v2, v2, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v19 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v3 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v19 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_max_f32_e32 v3, v3, v17 -; GFX9-NEXT: v_perm_b32 v3, v3, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v20 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v4 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v20 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_max_f32_e32 v4, v4, v17 -; GFX9-NEXT: v_perm_b32 v4, v4, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v21 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v5 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v21 -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_max_f32_e32 v5, v5, v17 -; GFX9-NEXT: v_perm_b32 v5, v5, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v22 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v6 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v22 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_max_f32_e32 v6, v6, v17 -; GFX9-NEXT: v_perm_b32 v6, v6, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v23 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v7 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v23 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_max_f32_e32 v7, v7, v17 -; GFX9-NEXT: v_perm_b32 v7, v7, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v24 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v8 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v24 -; GFX9-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX9-NEXT: v_max_f32_e32 v8, v8, v17 -; GFX9-NEXT: v_perm_b32 v8, v8, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v25 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v9 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v25 -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX9-NEXT: v_max_f32_e32 v9, v9, v17 -; GFX9-NEXT: v_perm_b32 v9, v9, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v26 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v10 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v26 -; GFX9-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX9-NEXT: v_max_f32_e32 v10, v10, v17 -; GFX9-NEXT: v_perm_b32 v10, v10, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v11 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v27 -; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX9-NEXT: v_max_f32_e32 v11, v11, v17 -; GFX9-NEXT: v_perm_b32 v11, v11, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v28 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v12 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v28 -; GFX9-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX9-NEXT: v_max_f32_e32 v12, v12, v17 -; GFX9-NEXT: v_perm_b32 v12, v12, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v29 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v13 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v29 -; GFX9-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX9-NEXT: v_max_f32_e32 v13, v13, v17 -; GFX9-NEXT: v_perm_b32 v13, v13, v16, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v30 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v14 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v30 -; GFX9-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX9-NEXT: v_max_f32_e32 v14, v14, v17 -; GFX9-NEXT: v_perm_b32 v14, v14, v16, s4 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v16, 16, v18 -; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v15 -; GFX9-NEXT: v_max_f32_e32 v16, v17, v16 -; GFX9-NEXT: v_and_b32_e32 v17, 0xffff0000, v18 -; GFX9-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX9-NEXT: v_max_f32_e32 v15, v15, v17 -; GFX9-NEXT: v_perm_b32 v15, v15, v16, s4 +; GFX9-NEXT: v_lshlrev_b32_e32 v17, 16, v16 +; GFX9-NEXT: v_lshlrev_b32_e32 v33, 16, v0 +; GFX9-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_max_f32_e32 v17, v33, v17 +; GFX9-NEXT: v_max_f32_e32 v0, v0, v16 +; GFX9-NEXT: v_perm_b32 v0, v0, v17, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v18, s4 +; GFX9-NEXT: v_perm_b32 v2, v2, v19, s4 +; GFX9-NEXT: v_perm_b32 v3, v3, v20, s4 +; GFX9-NEXT: v_perm_b32 v4, v4, v21, s4 +; GFX9-NEXT: v_perm_b32 v5, v5, v22, s4 +; GFX9-NEXT: v_perm_b32 v6, v6, v23, s4 +; GFX9-NEXT: v_perm_b32 v7, v7, v24, s4 +; GFX9-NEXT: v_perm_b32 v15, v15, v32, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_maxnum_v32bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v22 -; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 +; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 +; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v10 +; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 +; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v14 +; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 +; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 +; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v13 +; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 +; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 +; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_max_f32_e32 v39, v48, v39 +; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v17 +; GFX10-NEXT: v_max_f32_e32 v11, v11, v27 +; GFX10-NEXT: v_lshlrev_b32_e32 v27, 16, v1 ; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 -; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX10-NEXT: v_max_f32_e32 v5, v5, v21 -; GFX10-NEXT: v_max_f32_e32 v21, v53, v52 -; GFX10-NEXT: v_max_f32_e32 v6, v6, v22 -; GFX10-NEXT: v_max_f32_e32 v22, v55, v54 -; GFX10-NEXT: v_max_f32_e32 v7, v7, v23 -; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v25 -; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v9 +; GFX10-NEXT: v_max_f32_e32 v49, v50, v49 +; GFX10-NEXT: v_lshlrev_b32_e32 v50, 16, v16 +; GFX10-NEXT: v_max_f32_e32 v10, v10, v26 +; GFX10-NEXT: v_lshlrev_b32_e32 v26, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX10-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX10-NEXT: v_lshlrev_b32_e32 v52, 16, v9 ; GFX10-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 ; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX10-NEXT: v_max_f32_e32 v32, v33, v32 -; GFX10-NEXT: v_lshlrev_b32_e32 v33, 16, v10 -; GFX10-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 +; GFX10-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX10-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX10-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 +; GFX10-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX10-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX10-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX10-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX10-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX10-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX10-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_max_f32_e32 v33, v34, v33 +; GFX10-NEXT: v_lshlrev_b32_e32 v34, 16, v20 +; GFX10-NEXT: v_max_f32_e32 v14, v14, v30 +; GFX10-NEXT: v_lshlrev_b32_e32 v30, 16, v4 +; GFX10-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX10-NEXT: v_max_f32_e32 v35, v36, v35 +; GFX10-NEXT: v_lshlrev_b32_e32 v36, 16, v19 +; GFX10-NEXT: v_max_f32_e32 v13, v13, v29 +; GFX10-NEXT: v_lshlrev_b32_e32 v29, 16, v3 +; GFX10-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_max_f32_e32 v37, v38, v37 +; GFX10-NEXT: v_lshlrev_b32_e32 v38, 16, v18 +; GFX10-NEXT: v_max_f32_e32 v12, v12, v28 +; GFX10-NEXT: v_lshlrev_b32_e32 v28, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_max_f32_e32 v0, v0, v16 -; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v27 -; GFX10-NEXT: v_max_f32_e32 v34, v35, v34 -; GFX10-NEXT: v_lshlrev_b32_e32 v35, 16, v11 -; GFX10-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX10-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 ; GFX10-NEXT: v_max_f32_e32 v1, v1, v17 -; GFX10-NEXT: v_lshlrev_b32_e32 v17, 16, v28 -; GFX10-NEXT: v_max_f32_e32 v36, v37, v36 -; GFX10-NEXT: v_lshlrev_b32_e32 v37, 16, v12 -; GFX10-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX10-NEXT: v_max_f32_e32 v51, v52, v51 +; GFX10-NEXT: v_max_f32_e32 v9, v9, v25 +; GFX10-NEXT: v_max_f32_e32 v25, v54, v53 +; GFX10-NEXT: v_max_f32_e32 v8, v8, v24 +; GFX10-NEXT: v_max_f32_e32 v24, v64, v55 +; GFX10-NEXT: v_max_f32_e32 v7, v7, v23 +; GFX10-NEXT: v_max_f32_e32 v23, v66, v65 +; GFX10-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX10-NEXT: v_max_f32_e32 v22, v68, v67 +; GFX10-NEXT: v_max_f32_e32 v5, v5, v21 +; GFX10-NEXT: v_max_f32_e32 v21, v30, v34 +; GFX10-NEXT: v_max_f32_e32 v29, v29, v36 +; GFX10-NEXT: v_max_f32_e32 v28, v28, v38 +; GFX10-NEXT: v_max_f32_e32 v27, v27, v48 +; GFX10-NEXT: v_max_f32_e32 v26, v26, v50 ; GFX10-NEXT: v_max_f32_e32 v2, v2, v18 -; GFX10-NEXT: v_lshlrev_b32_e32 v18, 16, v29 -; GFX10-NEXT: v_max_f32_e32 v38, v39, v38 -; GFX10-NEXT: v_lshlrev_b32_e32 v39, 16, v13 -; GFX10-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 -; GFX10-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 ; GFX10-NEXT: v_max_f32_e32 v3, v3, v19 -; GFX10-NEXT: v_lshlrev_b32_e32 v19, 16, v30 -; GFX10-NEXT: v_max_f32_e32 v48, v49, v48 -; GFX10-NEXT: v_lshlrev_b32_e32 v49, 16, v14 -; GFX10-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX10-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 ; GFX10-NEXT: v_max_f32_e32 v4, v4, v20 -; GFX10-NEXT: v_lshlrev_b32_e32 v20, 16, v15 -; GFX10-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX10-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX10-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX10-NEXT: v_max_f32_e32 v50, v51, v50 -; GFX10-NEXT: v_max_f32_e32 v23, v65, v64 -; GFX10-NEXT: v_max_f32_e32 v8, v8, v24 -; GFX10-NEXT: v_max_f32_e32 v24, v67, v66 -; GFX10-NEXT: v_max_f32_e32 v9, v9, v25 -; GFX10-NEXT: v_max_f32_e32 v25, v33, v68 -; GFX10-NEXT: v_max_f32_e32 v10, v10, v26 -; GFX10-NEXT: v_max_f32_e32 v16, v35, v16 -; GFX10-NEXT: v_max_f32_e32 v11, v11, v27 -; GFX10-NEXT: v_max_f32_e32 v17, v37, v17 -; GFX10-NEXT: v_max_f32_e32 v12, v12, v28 -; GFX10-NEXT: v_max_f32_e32 v18, v39, v18 -; GFX10-NEXT: v_max_f32_e32 v13, v13, v29 -; GFX10-NEXT: v_max_f32_e32 v19, v49, v19 -; GFX10-NEXT: v_max_f32_e32 v14, v14, v30 -; GFX10-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v34, 0x7060302 -; GFX10-NEXT: v_perm_b32 v2, v2, v36, 0x7060302 -; GFX10-NEXT: v_perm_b32 v3, v3, v38, 0x7060302 -; GFX10-NEXT: v_perm_b32 v4, v4, v48, 0x7060302 -; GFX10-NEXT: v_perm_b32 v5, v5, v50, 0x7060302 -; GFX10-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX10-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX10-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX10-NEXT: v_perm_b32 v11, v11, v16, 0x7060302 -; GFX10-NEXT: v_perm_b32 v12, v12, v17, 0x7060302 -; GFX10-NEXT: v_perm_b32 v13, v13, v18, 0x7060302 -; GFX10-NEXT: v_perm_b32 v14, v14, v19, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v27, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v26, 0x7060302 +; GFX10-NEXT: v_perm_b32 v2, v2, v28, 0x7060302 +; GFX10-NEXT: v_perm_b32 v3, v3, v29, 0x7060302 +; GFX10-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX10-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX10-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 +; GFX10-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX10-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX10-NEXT: v_perm_b32 v9, v9, v51, 0x7060302 +; GFX10-NEXT: v_perm_b32 v10, v10, v49, 0x7060302 +; GFX10-NEXT: v_perm_b32 v11, v11, v39, 0x7060302 +; GFX10-NEXT: v_perm_b32 v12, v12, v37, 0x7060302 +; GFX10-NEXT: v_perm_b32 v13, v13, v35, 0x7060302 +; GFX10-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v21, 16, v31 -; GFX10-NEXT: v_and_b32_e32 v22, 0xffff0000, v31 -; GFX10-NEXT: v_max_f32_e32 v20, v20, v21 -; GFX10-NEXT: v_max_f32_e32 v15, v15, v22 -; GFX10-NEXT: v_perm_b32 v15, v15, v20, 0x7060302 +; GFX10-NEXT: v_lshlrev_b32_e32 v16, 16, v31 +; GFX10-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX10-NEXT: v_max_f32_e32 v16, v32, v16 +; GFX10-NEXT: v_max_f32_e32 v15, v15, v17 +; GFX10-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_maxnum_v32bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v26 -; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v10 -; GFX11-NEXT: v_and_b32_e32 v26, 0xffff0000, v26 -; GFX11-NEXT: v_and_b32_e32 v10, 0xffff0000, v10 -; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v27 -; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v50, 16, v21 -; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v23 -; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v24 -; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v8 -; GFX11-NEXT: v_and_b32_e32 v24, 0xffff0000, v24 -; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 -; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v5 -; GFX11-NEXT: v_dual_max_f32 v10, v10, v26 :: v_dual_and_b32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v25 +; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v17 +; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v16 +; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v0 +; GFX11-NEXT: v_and_b32_e32 v16, 0xffff0000, v16 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v54, 16, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v64, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: v_lshlrev_b32_e32 v65, 16, v22 +; GFX11-NEXT: v_lshlrev_b32_e32 v66, 16, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v11 +; GFX11-NEXT: v_dual_max_f32 v0, v0, v16 :: v_dual_and_b32 v11, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 +; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v21 +; GFX11-NEXT: v_lshlrev_b32_e32 v68, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v51, 16, v25 +; GFX11-NEXT: v_and_b32_e32 v21, 0xffff0000, v21 +; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v69, 16, v20 +; GFX11-NEXT: v_lshlrev_b32_e32 v70, 16, v4 +; GFX11-NEXT: v_and_b32_e32 v20, 0xffff0000, v20 +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v55, 16, v23 +; GFX11-NEXT: v_lshlrev_b32_e32 v71, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v25, 0xffff0000, v25 -; GFX11-NEXT: v_and_b32_e32 v27, 0xffff0000, v27 -; GFX11-NEXT: v_and_b32_e32 v11, 0xffff0000, v11 -; GFX11-NEXT: v_lshlrev_b32_e32 v80, 16, v28 -; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v22 +; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v52, 16, v9 +; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 +; GFX11-NEXT: v_lshlrev_b32_e32 v81, 16, v18 +; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v24 +; GFX11-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_and_b32 v24, 0xffff0000, v24 +; GFX11-NEXT: v_dual_max_f32 v5, v5, v21 :: v_dual_lshlrev_b32 v50, 16, v10 +; GFX11-NEXT: v_dual_max_f32 v21, v70, v69 :: v_dual_and_b32 v10, 0xffff0000, v10 +; GFX11-NEXT: v_dual_max_f32 v2, v2, v18 :: v_dual_max_f32 v3, v3, v19 +; GFX11-NEXT: v_dual_max_f32 v4, v4, v20 :: v_dual_lshlrev_b32 v49, 16, v26 +; GFX11-NEXT: v_dual_max_f32 v9, v9, v25 :: v_dual_and_b32 v26, 0xffff0000, v26 +; GFX11-NEXT: v_max_f32_e32 v6, v6, v22 +; GFX11-NEXT: v_dual_max_f32 v22, v68, v67 :: v_dual_lshlrev_b32 v37, 16, v28 ; GFX11-NEXT: v_and_b32_e32 v28, 0xffff0000, v28 -; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 -; GFX11-NEXT: v_lshlrev_b32_e32 v53, 16, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v82, 16, v29 -; GFX11-NEXT: v_lshlrev_b32_e32 v83, 16, v13 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_max_f32_e32 v10, v10, v26 +; GFX11-NEXT: v_max_f32_e32 v26, v52, v51 +; GFX11-NEXT: v_perm_b32 v4, v4, v21, 0x7060302 +; GFX11-NEXT: v_max_f32_e32 v25, v54, v53 +; GFX11-NEXT: v_perm_b32 v5, v5, v22, 0x7060302 +; GFX11-NEXT: v_perm_b32 v9, v9, v26, 0x7060302 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 ; GFX11-NEXT: v_and_b32_e32 v23, 0xffff0000, v23 -; GFX11-NEXT: v_and_b32_e32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v31 +; GFX11-NEXT: v_and_b32_e32 v8, 0xffff0000, v8 +; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v13 ; GFX11-NEXT: v_and_b32_e32 v13, 0xffff0000, v13 -; GFX11-NEXT: v_lshlrev_b32_e32 v84, 16, v30 -; GFX11-NEXT: v_lshlrev_b32_e32 v85, 16, v14 -; GFX11-NEXT: v_and_b32_e32 v22, 0xffff0000, v22 -; GFX11-NEXT: v_and_b32_e32 v30, 0xffff0000, v30 -; GFX11-NEXT: v_and_b32_e32 v14, 0xffff0000, v14 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX11-NEXT: v_lshlrev_b32_e32 v86, 16, v15 -; GFX11-NEXT: v_lshlrev_b32_e32 v67, 16, v9 -; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v9 -; GFX11-NEXT: v_lshlrev_b32_e32 v48, 16, v20 -; GFX11-NEXT: v_dual_max_f32 v11, v11, v27 :: v_dual_and_b32 v20, 0xffff0000, v20 -; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 -; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX11-NEXT: v_dual_max_f32 v26, v71, v70 :: v_dual_lshlrev_b32 v49, 16, v4 -; GFX11-NEXT: v_dual_max_f32 v13, v13, v29 :: v_dual_and_b32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v37, 16, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v19 +; GFX11-NEXT: v_lshlrev_b32_e32 v39, 16, v27 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_max_f32_e32 v4, v4, v20 -; GFX11-NEXT: v_dual_max_f32 v8, v8, v24 :: v_dual_max_f32 v9, v9, v25 -; GFX11-NEXT: v_max_f32_e32 v25, v69, v68 -; GFX11-NEXT: v_dual_max_f32 v20, v51, v50 :: v_dual_lshlrev_b32 v39, 16, v3 -; GFX11-NEXT: v_max_f32_e32 v27, v81, v80 -; GFX11-NEXT: v_max_f32_e32 v12, v12, v28 -; GFX11-NEXT: v_dual_max_f32 v28, v83, v82 :: v_dual_max_f32 v29, v85, v84 -; GFX11-NEXT: v_dual_max_f32 v6, v6, v22 :: v_dual_and_b32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_max_f32_e32 v22, v55, v54 -; GFX11-NEXT: v_lshlrev_b32_e32 v36, 16, v18 -; GFX11-NEXT: v_lshlrev_b32_e32 v34, 16, v17 -; GFX11-NEXT: v_and_b32_e32 v17, 0xffff0000, v17 -; GFX11-NEXT: v_and_b32_e32 v18, 0xffff0000, v18 +; GFX11-NEXT: v_dual_max_f32 v8, v8, v24 :: v_dual_and_b32 v27, 0xffff0000, v27 +; GFX11-NEXT: v_max_f32_e32 v24, v64, v55 +; GFX11-NEXT: v_lshlrev_b32_e32 v38, 16, v12 +; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v12 +; GFX11-NEXT: v_lshlrev_b32_e32 v35, 16, v29 +; GFX11-NEXT: v_max_f32_e32 v7, v7, v23 +; GFX11-NEXT: v_max_f32_e32 v23, v66, v65 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_max_f32 v12, v12, v28 :: v_dual_and_b32 v29, 0xffff0000, v29 +; GFX11-NEXT: v_dual_max_f32 v28, v48, v39 :: v_dual_lshlrev_b32 v33, 16, v30 +; GFX11-NEXT: v_dual_max_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v34, 16, v14 +; GFX11-NEXT: v_lshlrev_b32_e32 v32, 16, v15 +; GFX11-NEXT: v_dual_max_f32 v11, v11, v27 :: v_dual_and_b32 v14, 0xffff0000, v14 +; GFX11-NEXT: v_dual_max_f32 v27, v50, v49 :: v_dual_and_b32 v30, 0xffff0000, v30 +; GFX11-NEXT: v_max_f32_e32 v29, v38, v37 +; GFX11-NEXT: v_and_b32_e32 v15, 0xffff0000, v15 +; GFX11-NEXT: v_max_f32_e32 v37, v86, v85 +; GFX11-NEXT: v_perm_b32 v6, v6, v23, 0x7060302 ; GFX11-NEXT: v_max_f32_e32 v14, v14, v30 -; GFX11-NEXT: v_dual_max_f32 v7, v7, v23 :: v_dual_and_b32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_max_f32_e32 v23, v65, v64 -; GFX11-NEXT: v_and_b32_e32 v19, 0xffff0000, v19 -; GFX11-NEXT: v_dual_max_f32 v24, v67, v66 :: v_dual_and_b32 v21, 0xffff0000, v21 -; GFX11-NEXT: v_max_f32_e32 v2, v2, v18 -; GFX11-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v32, 16, v16 -; GFX11-NEXT: v_max_f32_e32 v18, v39, v38 -; GFX11-NEXT: v_dual_max_f32 v3, v3, v19 :: v_dual_and_b32 v16, 0xffff0000, v16 -; GFX11-NEXT: v_max_f32_e32 v19, v49, v48 -; GFX11-NEXT: v_max_f32_e32 v17, v37, v36 -; GFX11-NEXT: v_lshlrev_b32_e32 v33, 16, v0 -; GFX11-NEXT: v_dual_max_f32 v5, v5, v21 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_max_f32_e32 v21, v53, v52 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v2, v2, v17, 0x7060302 -; GFX11-NEXT: v_perm_b32 v3, v3, v18, 0x7060302 -; GFX11-NEXT: v_max_f32_e32 v0, v0, v16 -; GFX11-NEXT: v_max_f32_e32 v16, v35, v34 -; GFX11-NEXT: v_max_f32_e32 v32, v33, v32 -; GFX11-NEXT: v_perm_b32 v4, v4, v19, 0x7060302 -; GFX11-NEXT: v_perm_b32 v5, v5, v20, 0x7060302 -; GFX11-NEXT: v_perm_b32 v6, v6, v21, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v16, 0x7060302 -; GFX11-NEXT: v_perm_b32 v0, v0, v32, 0x7060302 -; GFX11-NEXT: v_perm_b32 v7, v7, v22, 0x7060302 -; GFX11-NEXT: v_perm_b32 v8, v8, v23, 0x7060302 -; GFX11-NEXT: v_perm_b32 v9, v9, v24, 0x7060302 -; GFX11-NEXT: v_perm_b32 v10, v10, v25, 0x7060302 -; GFX11-NEXT: v_perm_b32 v11, v11, v26, 0x7060302 -; GFX11-NEXT: v_perm_b32 v12, v12, v27, 0x7060302 -; GFX11-NEXT: v_perm_b32 v13, v13, v28, 0x7060302 -; GFX11-NEXT: v_perm_b32 v14, v14, v29, 0x7060302 -; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v16, 16, v31 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_max_f32 v16, v86, v16 :: v_dual_and_b32 v17, 0xffff0000, v31 -; GFX11-NEXT: v_max_f32_e32 v15, v15, v17 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_dual_max_f32 v30, v36, v35 :: v_dual_max_f32 v33, v34, v33 +; GFX11-NEXT: v_dual_max_f32 v34, v80, v71 :: v_dual_max_f32 v35, v82, v81 +; GFX11-NEXT: v_max_f32_e32 v36, v84, v83 +; GFX11-NEXT: v_dual_max_f32 v16, v32, v16 :: v_dual_max_f32 v15, v15, v17 +; GFX11-NEXT: v_perm_b32 v0, v0, v37, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_perm_b32 v2, v2, v35, 0x7060302 +; GFX11-NEXT: v_perm_b32 v1, v1, v36, 0x7060302 +; GFX11-NEXT: v_perm_b32 v3, v3, v34, 0x7060302 +; GFX11-NEXT: v_perm_b32 v7, v7, v24, 0x7060302 +; GFX11-NEXT: v_perm_b32 v8, v8, v25, 0x7060302 +; GFX11-NEXT: v_perm_b32 v10, v10, v27, 0x7060302 +; GFX11-NEXT: v_perm_b32 v11, v11, v28, 0x7060302 +; GFX11-NEXT: v_perm_b32 v12, v12, v29, 0x7060302 +; GFX11-NEXT: v_perm_b32 v13, v13, v30, 0x7060302 +; GFX11-NEXT: v_perm_b32 v14, v14, v33, 0x7060302 ; GFX11-NEXT: v_perm_b32 v15, v15, v16, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <32 x bfloat> @llvm.maxnum.v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) @@ -21043,13 +20861,13 @@ define <3 x i16> @v_fptosi_v3bf16_to_v3i16(<3 x bfloat> %x) { ; GFX8-LABEL: v_fptosi_v3bf16_to_v3i16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD -; GFX8-NEXT: v_cvt_i32_f32_e32 v0, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_cvt_i32_f32_e32 v2, v2 +; GFX8-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX8-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fptosi_v3bf16_to_v3i16: @@ -21140,65 +20958,65 @@ define <4 x i16> @v_fptosi_v4bf16_to_v4i16(<4 x bfloat> %x) { ; GFX8-LABEL: v_fptosi_v4bf16_to_v4i16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v0 -; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v1 -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_cvt_i32_f32_sdwa v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD -; GFX8-NEXT: v_cvt_i32_f32_e32 v0, v0 -; GFX8-NEXT: v_cvt_i32_f32_sdwa v3, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD -; GFX8-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_cvt_i32_f32_e32 v2, v2 +; GFX8-NEXT: v_cvt_i32_f32_e32 v3, v3 +; GFX8-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD +; GFX8-NEXT: v_cvt_i32_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fptosi_v4bf16_to_v4i16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v0 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v1 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v1 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0 ; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 +; GFX9-NEXT: v_cvt_i32_f32_e32 v0, v0 ; GFX9-NEXT: v_cvt_i32_f32_e32 v1, v1 ; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4 -; GFX9-NEXT: v_perm_b32 v1, v1, v3, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fptosi_v4bf16_to_v4i16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0 ; GFX10-NEXT: v_cvt_i32_f32_e32 v3, v3 +; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0 ; GFX10-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x5040100 +; GFX10-NEXT: v_perm_b32 v0, v0, v3, 0x5040100 +; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fptosi_v4bf16_to_v4i16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v1 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX11-NEXT: v_cvt_i32_f32_e32 v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cvt_i32_f32_e32 v3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_cvt_i32_f32_e32 v0, v0 ; GFX11-NEXT: v_cvt_i32_f32_e32 v1, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x5040100 -; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x5040100 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v0, v3, 0x5040100 +; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x5040100 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = fptosi <4 x bfloat> %x to <4 x i16> ret <4 x i16> %op @@ -22538,13 +22356,11 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) { ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX8-NEXT: v_cvt_f32_i32_sdwa v3, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX8-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_sitofp_v3i16_to_v3bf16: @@ -22552,11 +22368,10 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) { ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 ; GFX9-NEXT: v_perm_b32 v0, v2, v0, s4 -; GFX9-NEXT: v_perm_b32 v1, v3, v1, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_sitofp_v3i16_to_v3bf16: @@ -22564,29 +22379,10 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) { ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_sitofp_v3i16_to_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_ashrrev_i32_e32 v2, 16, v0 -; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16 -; GFX11-NEXT: v_ashrrev_i32_e32 v3, 16, v1 -; GFX11-NEXT: v_bfe_i32 v1, v1, 0, 16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cvt_f32_i32_e32 v3, v3 -; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v3, v1, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = sitofp <3 x i16> %x to <3 x bfloat> ret <3 x bfloat> %op } @@ -22629,55 +22425,55 @@ define <4 x bfloat> @v_sitofp_v4i16_to_v4bf16(<4 x i16> %x) { ; GFX8-LABEL: v_sitofp_v4i16_to_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX8-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX8-NEXT: v_cvt_f32_i32_sdwa v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX8-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX8-NEXT: v_cvt_f32_i32_sdwa v3, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX8-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_alignbit_b32 v0, v3, v0, 16 ; GFX8-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_sitofp_v4i16_to_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX9-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX9-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-NEXT: v_cvt_f32_i32_sdwa v3, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX9-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v2, v0, s4 -; GFX9-NEXT: v_perm_b32 v1, v3, v1, s4 +; GFX9-NEXT: v_perm_b32 v0, v3, v0, s4 +; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_sitofp_v4i16_to_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-NEXT: v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX10-NEXT: v_cvt_f32_i32_sdwa v0, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX10-NEXT: v_cvt_f32_i32_sdwa v3, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX10-NEXT: v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v3, v0, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_sitofp_v4i16_to_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_ashrrev_i32_e32 v2, 16, v0 +; GFX11-NEXT: v_ashrrev_i32_e32 v2, 16, v1 +; GFX11-NEXT: v_ashrrev_i32_e32 v3, 16, v0 ; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16 -; GFX11-NEXT: v_ashrrev_i32_e32 v3, 16, v1 ; GFX11-NEXT: v_bfe_i32 v1, v1, 0, 16 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cvt_f32_i32_e32 v3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v3, v1, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v3, v0, 0x7060302 +; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = sitofp <4 x i16> %x to <4 x bfloat> ret <4 x bfloat> %op @@ -22813,12 +22609,12 @@ define <3 x bfloat> @v_sitofp_v3i32_to_v3bf16(<3 x i32> %x) { ; GFX8-LABEL: v_sitofp_v3i32_to_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX8-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16 +; GFX8-NEXT: v_cvt_f32_i32_e32 v3, v1 +; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3 +; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_sitofp_v3i32_to_v3bf16: @@ -22875,22 +22671,22 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) { ; GFX8-LABEL: v_sitofp_v4i32_to_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_cvt_f32_i32_e32 v3, v3 ; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1 ; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX8-NEXT: v_cvt_f32_i32_e32 v3, v3 ; GFX8-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_sitofp_v4i32_to_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v2 ; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v2 ; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v3 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 ; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 @@ -22900,9 +22696,9 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) { ; GFX10-LABEL: v_sitofp_v4i32_to_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2 ; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2 ; GFX10-NEXT: v_cvt_f32_i32_e32 v3, v3 ; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x7060302 @@ -22911,11 +22707,11 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) { ; GFX11-LABEL: v_sitofp_v4i32_to_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2 ; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2 ; GFX11-NEXT: v_cvt_f32_i32_e32 v3, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] @@ -23330,130 +23126,130 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) { ; GFX8-LABEL: v_sitofp_v3i64_to_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_xor_b32_e32 v7, v0, v1 -; GFX8-NEXT: v_ffbh_i32_e32 v6, v1 +; GFX8-NEXT: v_xor_b32_e32 v7, v4, v5 +; GFX8-NEXT: v_ffbh_i32_e32 v6, v5 ; GFX8-NEXT: v_ashrrev_i32_e32 v7, 31, v7 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, -1, v6 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 32, v7 ; GFX8-NEXT: v_min_u32_e32 v6, v6, v7 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] -; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX8-NEXT: v_xor_b32_e32 v1, v2, v3 -; GFX8-NEXT: v_cvt_f32_i32_e32 v7, v0 -; GFX8-NEXT: v_ffbh_i32_e32 v0, v3 -; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GFX8-NEXT: v_add_u32_e32 v0, vcc, -1, v0 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, 32, v1 -; GFX8-NEXT: v_min_u32_e32 v8, v0, v1 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6 -; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5] +; GFX8-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX8-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX8-NEXT: v_cvt_f32_i32_e32 v4, v4 +; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v6 +; GFX8-NEXT: v_ldexp_f32 v6, v4, v5 +; GFX8-NEXT: v_xor_b32_e32 v5, v0, v1 +; GFX8-NEXT: v_ffbh_i32_e32 v4, v1 +; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v5 +; GFX8-NEXT: v_add_u32_e32 v4, vcc, -1, v4 +; GFX8-NEXT: v_add_u32_e32 v5, vcc, 32, v5 +; GFX8-NEXT: v_min_u32_e32 v7, v4, v5 +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v7, v[0:1] +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v6 +; GFX8-NEXT: v_min_u32_e32 v0, 1, v4 +; GFX8-NEXT: v_or_b32_e32 v0, v5, v0 +; GFX8-NEXT: v_xor_b32_e32 v5, v2, v3 +; GFX8-NEXT: v_ffbh_i32_e32 v4, v3 +; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v5 +; GFX8-NEXT: v_add_u32_e32 v4, vcc, -1, v4 +; GFX8-NEXT: v_add_u32_e32 v5, vcc, 32, v5 +; GFX8-NEXT: v_min_u32_e32 v4, v4, v5 +; GFX8-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3] ; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v8 -; GFX8-NEXT: v_ldexp_f32 v2, v7, v2 -; GFX8-NEXT: v_ldexp_f32 v3, v0, v1 -; GFX8-NEXT: v_xor_b32_e32 v1, v4, v5 -; GFX8-NEXT: v_ffbh_i32_e32 v0, v5 -; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GFX8-NEXT: v_add_u32_e32 v0, vcc, -1, v0 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, 32, v1 -; GFX8-NEXT: v_min_u32_e32 v6, v0, v1 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5] -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v3, v2, 16 -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6 -; GFX8-NEXT: v_ldexp_f32 v1, v1, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_min_u32_e32 v2, 1, v2 +; GFX8-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX8-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v4 +; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v7 +; GFX8-NEXT: v_ldexp_f32 v2, v2, v3 +; GFX8-NEXT: v_ldexp_f32 v0, v0, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_sitofp_v3i64_to_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_xor_b32_e32 v7, v4, v5 +; GFX9-NEXT: v_ffbh_i32_e32 v6, v5 +; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v7 +; GFX9-NEXT: v_add_u32_e32 v6, -1, v6 +; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 +; GFX9-NEXT: v_min_u32_e32 v6, v6, v7 +; GFX9-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5] ; GFX9-NEXT: v_xor_b32_e32 v7, v0, v1 +; GFX9-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX9-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX9-NEXT: v_sub_u32_e32 v5, 32, v6 ; GFX9-NEXT: v_ffbh_i32_e32 v6, v1 ; GFX9-NEXT: v_ashrrev_i32_e32 v7, 31, v7 ; GFX9-NEXT: v_add_u32_e32 v6, -1, v6 ; GFX9-NEXT: v_add_u32_e32 v7, 32, v7 +; GFX9-NEXT: v_cvt_f32_i32_e32 v4, v4 ; GFX9-NEXT: v_min_u32_e32 v6, v6, v7 ; GFX9-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] -; GFX9-NEXT: v_sub_u32_e32 v6, 32, v6 +; GFX9-NEXT: s_mov_b32 s4, 0x7060302 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 +; GFX9-NEXT: v_ldexp_f32 v4, v4, v5 +; GFX9-NEXT: v_or_b32_e32 v5, v1, v0 ; GFX9-NEXT: v_xor_b32_e32 v1, v2, v3 -; GFX9-NEXT: v_cvt_f32_i32_e32 v7, v0 ; GFX9-NEXT: v_ffbh_i32_e32 v0, v3 ; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v1 ; GFX9-NEXT: v_add_u32_e32 v0, -1, v0 ; GFX9-NEXT: v_add_u32_e32 v1, 32, v1 -; GFX9-NEXT: v_min_u32_e32 v8, v0, v1 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] -; GFX9-NEXT: v_ldexp_f32 v2, v7, v6 -; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX9-NEXT: v_xor_b32_e32 v1, v4, v5 -; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v0 -; GFX9-NEXT: v_ffbh_i32_e32 v0, v5 -; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v1 -; GFX9-NEXT: v_add_u32_e32 v0, -1, v0 -; GFX9-NEXT: v_add_u32_e32 v1, 32, v1 ; GFX9-NEXT: v_min_u32_e32 v7, v0, v1 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[4:5] -; GFX9-NEXT: v_sub_u32_e32 v6, 32, v8 +; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3] +; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v5 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v0 -; GFX9-NEXT: v_ldexp_f32 v3, v3, v6 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v3, v2, s4 +; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0 +; GFX9-NEXT: v_sub_u32_e32 v1, 32, v6 +; GFX9-NEXT: v_ldexp_f32 v1, v2, v1 ; GFX9-NEXT: v_sub_u32_e32 v2, 32, v7 -; GFX9-NEXT: v_ldexp_f32 v1, v1, v2 -; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 +; GFX9-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v4, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_sitofp_v3i64_to_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_xor_b32_e32 v7, v0, v1 -; GFX10-NEXT: v_xor_b32_e32 v8, v2, v3 -; GFX10-NEXT: v_xor_b32_e32 v9, v4, v5 -; GFX10-NEXT: v_ffbh_i32_e32 v6, v1 -; GFX10-NEXT: v_ffbh_i32_e32 v10, v3 +; GFX10-NEXT: v_xor_b32_e32 v8, v4, v5 +; GFX10-NEXT: v_xor_b32_e32 v9, v2, v3 +; GFX10-NEXT: v_ffbh_i32_e32 v6, v5 +; GFX10-NEXT: v_ffbh_i32_e32 v10, v1 ; GFX10-NEXT: v_ashrrev_i32_e32 v7, 31, v7 -; GFX10-NEXT: v_ashrrev_i32_e32 v8, 31, v8 -; GFX10-NEXT: v_ffbh_i32_e32 v11, v5 +; GFX10-NEXT: v_ffbh_i32_e32 v11, v3 ; GFX10-NEXT: v_ashrrev_i32_e32 v9, 31, v9 +; GFX10-NEXT: v_ashrrev_i32_e32 v8, 31, v8 ; GFX10-NEXT: v_add_nc_u32_e32 v6, -1, v6 -; GFX10-NEXT: v_add_nc_u32_e32 v7, 32, v7 ; GFX10-NEXT: v_add_nc_u32_e32 v10, -1, v10 -; GFX10-NEXT: v_add_nc_u32_e32 v8, 32, v8 +; GFX10-NEXT: v_add_nc_u32_e32 v7, 32, v7 ; GFX10-NEXT: v_add_nc_u32_e32 v11, -1, v11 ; GFX10-NEXT: v_add_nc_u32_e32 v9, 32, v9 -; GFX10-NEXT: v_min_u32_e32 v6, v6, v7 -; GFX10-NEXT: v_min_u32_e32 v7, v10, v8 -; GFX10-NEXT: v_min_u32_e32 v8, v11, v9 -; GFX10-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] +; GFX10-NEXT: v_add_nc_u32_e32 v8, 32, v8 +; GFX10-NEXT: v_min_u32_e32 v7, v10, v7 +; GFX10-NEXT: v_min_u32_e32 v9, v11, v9 +; GFX10-NEXT: v_min_u32_e32 v6, v6, v8 +; GFX10-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1] +; GFX10-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3] +; GFX10-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5] ; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v6 -; GFX10-NEXT: v_lshlrev_b64 v[2:3], v7, v[2:3] -; GFX10-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] ; GFX10-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX10-NEXT: v_min_u32_e32 v2, 1, v2 ; GFX10-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX10-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX10-NEXT: v_or_b32_e32 v1, v3, v2 -; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v7 ; GFX10-NEXT: v_or_b32_e32 v2, v5, v4 -; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v8 +; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v7 +; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v9 ; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1 ; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2 -; GFX10-NEXT: v_ldexp_f32 v0, v0, v6 -; GFX10-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX10-NEXT: v_ldexp_f32 v2, v2, v4 +; GFX10-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX10-NEXT: v_ldexp_f32 v1, v1, v4 +; GFX10-NEXT: v_ldexp_f32 v2, v2, v6 ; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX10-NEXT: v_alignbit_b32 v1, s4, v2, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] @@ -23579,236 +23375,231 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) { ; GFX8-LABEL: v_sitofp_v4i64_to_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_xor_b32_e32 v9, v0, v1 -; GFX8-NEXT: v_ffbh_i32_e32 v8, v1 +; GFX8-NEXT: v_xor_b32_e32 v9, v4, v5 +; GFX8-NEXT: v_ffbh_i32_e32 v8, v5 ; GFX8-NEXT: v_ashrrev_i32_e32 v9, 31, v9 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, -1, v8 ; GFX8-NEXT: v_add_u32_e32 v9, vcc, 32, v9 ; GFX8-NEXT: v_min_u32_e32 v8, v8, v9 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX8-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX8-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX8-NEXT: v_xor_b32_e32 v5, v6, v7 +; GFX8-NEXT: v_cvt_f32_i32_e32 v9, v4 +; GFX8-NEXT: v_ffbh_i32_e32 v4, v7 +; GFX8-NEXT: v_ashrrev_i32_e32 v5, 31, v5 +; GFX8-NEXT: v_add_u32_e32 v4, vcc, -1, v4 +; GFX8-NEXT: v_add_u32_e32 v5, vcc, 32, v5 +; GFX8-NEXT: v_min_u32_e32 v10, v4, v5 +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7] +; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v8 +; GFX8-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX8-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX8-NEXT: v_cvt_f32_i32_e32 v4, v4 +; GFX8-NEXT: v_ldexp_f32 v5, v9, v6 +; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v10 +; GFX8-NEXT: v_xor_b32_e32 v7, v0, v1 +; GFX8-NEXT: v_ldexp_f32 v4, v4, v6 +; GFX8-NEXT: v_ffbh_i32_e32 v6, v1 +; GFX8-NEXT: v_ashrrev_i32_e32 v7, 31, v7 +; GFX8-NEXT: v_add_u32_e32 v6, vcc, -1, v6 +; GFX8-NEXT: v_add_u32_e32 v7, vcc, 32, v7 +; GFX8-NEXT: v_min_u32_e32 v6, v6, v7 +; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 ; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX8-NEXT: v_xor_b32_e32 v1, v2, v3 -; GFX8-NEXT: v_cvt_f32_i32_e32 v9, v0 +; GFX8-NEXT: v_cvt_f32_i32_e32 v7, v0 ; GFX8-NEXT: v_ffbh_i32_e32 v0, v3 ; GFX8-NEXT: v_ashrrev_i32_e32 v1, 31, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, -1, v0 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, 32, v1 -; GFX8-NEXT: v_min_u32_e32 v10, v0, v1 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v10, v[2:3] -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v8 +; GFX8-NEXT: v_min_u32_e32 v8, v0, v1 +; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] +; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6 ; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX8-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX8-NEXT: v_ldexp_f32 v3, v9, v2 -; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v10 -; GFX8-NEXT: v_xor_b32_e32 v2, v4, v5 -; GFX8-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX8-NEXT: v_ffbh_i32_e32 v1, v5 -; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v2 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, -1, v1 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, 32, v2 -; GFX8-NEXT: v_min_u32_e32 v8, v1, v2 -; GFX8-NEXT: v_lshlrev_b64 v[1:2], v8, v[4:5] +; GFX8-NEXT: v_ldexp_f32 v1, v7, v2 +; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v8 +; GFX8-NEXT: v_ldexp_f32 v0, v0, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX8-NEXT: v_or_b32_e32 v1, v2, v1 -; GFX8-NEXT: v_xor_b32_e32 v2, v6, v7 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 -; GFX8-NEXT: v_cvt_f32_i32_e32 v3, v1 -; GFX8-NEXT: v_ffbh_i32_e32 v1, v7 -; GFX8-NEXT: v_ashrrev_i32_e32 v2, 31, v2 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, -1, v1 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, 32, v2 -; GFX8-NEXT: v_min_u32_e32 v4, v1, v2 -; GFX8-NEXT: v_lshlrev_b64 v[1:2], v4, v[6:7] -; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v8 -; GFX8-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX8-NEXT: v_or_b32_e32 v1, v2, v1 -; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX8-NEXT: v_ldexp_f32 v2, v3, v5 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v4 -; GFX8-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v4, v5, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_sitofp_v4i64_to_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_xor_b32_e32 v9, v0, v1 -; GFX9-NEXT: v_ffbh_i32_e32 v8, v1 +; GFX9-NEXT: v_xor_b32_e32 v9, v4, v5 +; GFX9-NEXT: v_ffbh_i32_e32 v8, v5 ; GFX9-NEXT: v_ashrrev_i32_e32 v9, 31, v9 ; GFX9-NEXT: v_add_u32_e32 v8, -1, v8 ; GFX9-NEXT: v_add_u32_e32 v9, 32, v9 ; GFX9-NEXT: v_min_u32_e32 v8, v8, v9 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 +; GFX9-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX9-NEXT: v_sub_u32_e32 v8, 32, v8 +; GFX9-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX9-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX9-NEXT: v_xor_b32_e32 v5, v6, v7 +; GFX9-NEXT: v_cvt_f32_i32_e32 v9, v4 +; GFX9-NEXT: v_ffbh_i32_e32 v4, v7 +; GFX9-NEXT: v_ashrrev_i32_e32 v5, 31, v5 +; GFX9-NEXT: v_add_u32_e32 v4, -1, v4 +; GFX9-NEXT: v_add_u32_e32 v5, 32, v5 +; GFX9-NEXT: v_min_u32_e32 v10, v4, v5 +; GFX9-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7] +; GFX9-NEXT: v_ldexp_f32 v6, v9, v8 +; GFX9-NEXT: v_xor_b32_e32 v8, v0, v1 +; GFX9-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX9-NEXT: v_ffbh_i32_e32 v7, v1 +; GFX9-NEXT: v_ashrrev_i32_e32 v8, 31, v8 +; GFX9-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX9-NEXT: v_add_u32_e32 v7, -1, v7 +; GFX9-NEXT: v_add_u32_e32 v8, 32, v8 +; GFX9-NEXT: v_cvt_f32_i32_e32 v4, v4 +; GFX9-NEXT: v_min_u32_e32 v7, v7, v8 +; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1] +; GFX9-NEXT: v_sub_u32_e32 v5, 32, v10 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX9-NEXT: v_or_b32_e32 v9, v1, v0 +; GFX9-NEXT: v_ldexp_f32 v4, v4, v5 +; GFX9-NEXT: v_or_b32_e32 v5, v1, v0 ; GFX9-NEXT: v_xor_b32_e32 v1, v2, v3 ; GFX9-NEXT: v_ffbh_i32_e32 v0, v3 ; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v1 ; GFX9-NEXT: v_add_u32_e32 v0, -1, v0 ; GFX9-NEXT: v_add_u32_e32 v1, 32, v1 -; GFX9-NEXT: v_min_u32_e32 v10, v0, v1 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v10, v[2:3] -; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v9 +; GFX9-NEXT: v_min_u32_e32 v8, v0, v1 +; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] +; GFX9-NEXT: v_cvt_f32_i32_e32 v2, v5 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX9-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX9-NEXT: v_sub_u32_e32 v1, 32, v8 -; GFX9-NEXT: v_ldexp_f32 v3, v2, v1 -; GFX9-NEXT: v_sub_u32_e32 v1, 32, v10 -; GFX9-NEXT: v_xor_b32_e32 v2, v4, v5 -; GFX9-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX9-NEXT: v_ffbh_i32_e32 v1, v5 -; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v2 -; GFX9-NEXT: v_add_u32_e32 v1, -1, v1 -; GFX9-NEXT: v_add_u32_e32 v2, 32, v2 -; GFX9-NEXT: v_min_u32_e32 v8, v1, v2 -; GFX9-NEXT: v_lshlrev_b64 v[1:2], v8, v[4:5] -; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 -; GFX9-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX9-NEXT: v_or_b32_e32 v3, v2, v1 -; GFX9-NEXT: v_xor_b32_e32 v2, v6, v7 -; GFX9-NEXT: v_ffbh_i32_e32 v1, v7 -; GFX9-NEXT: v_ashrrev_i32_e32 v2, 31, v2 -; GFX9-NEXT: v_add_u32_e32 v1, -1, v1 -; GFX9-NEXT: v_add_u32_e32 v2, 32, v2 -; GFX9-NEXT: v_min_u32_e32 v4, v1, v2 -; GFX9-NEXT: v_lshlrev_b64 v[1:2], v4, v[6:7] -; GFX9-NEXT: v_cvt_f32_i32_e32 v3, v3 -; GFX9-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX9-NEXT: v_or_b32_e32 v1, v2, v1 -; GFX9-NEXT: v_cvt_f32_i32_e32 v1, v1 +; GFX9-NEXT: v_sub_u32_e32 v1, 32, v7 +; GFX9-NEXT: v_ldexp_f32 v1, v2, v1 ; GFX9-NEXT: v_sub_u32_e32 v2, 32, v8 -; GFX9-NEXT: v_ldexp_f32 v2, v3, v2 -; GFX9-NEXT: v_sub_u32_e32 v3, 32, v4 -; GFX9-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX9-NEXT: s_mov_b32 s4, 0x7060302 +; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4 +; GFX9-NEXT: v_perm_b32 v1, v4, v6, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_sitofp_v4i64_to_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_xor_b32_e32 v8, v0, v1 -; GFX10-NEXT: v_ffbh_i32_e32 v9, v1 -; GFX10-NEXT: v_ffbh_i32_e32 v10, v3 -; GFX10-NEXT: v_xor_b32_e32 v11, v2, v3 -; GFX10-NEXT: v_xor_b32_e32 v13, v4, v5 +; GFX10-NEXT: v_xor_b32_e32 v8, v4, v5 +; GFX10-NEXT: v_ffbh_i32_e32 v9, v5 +; GFX10-NEXT: v_xor_b32_e32 v11, v6, v7 +; GFX10-NEXT: v_xor_b32_e32 v13, v0, v1 +; GFX10-NEXT: v_xor_b32_e32 v14, v2, v3 ; GFX10-NEXT: v_ashrrev_i32_e32 v8, 31, v8 ; GFX10-NEXT: v_add_nc_u32_e32 v9, -1, v9 -; GFX10-NEXT: v_xor_b32_e32 v15, v6, v7 -; GFX10-NEXT: v_ffbh_i32_e32 v12, v5 -; GFX10-NEXT: v_ffbh_i32_e32 v14, v7 -; GFX10-NEXT: v_add_nc_u32_e32 v8, 32, v8 +; GFX10-NEXT: v_ffbh_i32_e32 v10, v7 +; GFX10-NEXT: v_ffbh_i32_e32 v12, v1 ; GFX10-NEXT: v_ashrrev_i32_e32 v11, 31, v11 +; GFX10-NEXT: v_add_nc_u32_e32 v8, 32, v8 +; GFX10-NEXT: v_ashrrev_i32_e32 v14, 31, v14 +; GFX10-NEXT: v_add_nc_u32_e32 v10, -1, v10 ; GFX10-NEXT: v_add_nc_u32_e32 v12, -1, v12 -; GFX10-NEXT: v_add_nc_u32_e32 v14, -1, v14 -; GFX10-NEXT: v_min_u32_e32 v8, v9, v8 -; GFX10-NEXT: v_add_nc_u32_e32 v9, -1, v10 -; GFX10-NEXT: v_ashrrev_i32_e32 v10, 31, v13 -; GFX10-NEXT: v_ashrrev_i32_e32 v13, 31, v15 ; GFX10-NEXT: v_add_nc_u32_e32 v11, 32, v11 -; GFX10-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] -; GFX10-NEXT: v_add_nc_u32_e32 v10, 32, v10 -; GFX10-NEXT: v_add_nc_u32_e32 v13, 32, v13 -; GFX10-NEXT: v_min_u32_e32 v9, v9, v11 +; GFX10-NEXT: v_min_u32_e32 v8, v9, v8 +; GFX10-NEXT: v_ashrrev_i32_e32 v9, 31, v13 +; GFX10-NEXT: v_ffbh_i32_e32 v13, v3 +; GFX10-NEXT: v_add_nc_u32_e32 v14, 32, v14 +; GFX10-NEXT: v_min_u32_e32 v10, v10, v11 +; GFX10-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX10-NEXT: v_add_nc_u32_e32 v9, 32, v9 +; GFX10-NEXT: v_add_nc_u32_e32 v13, -1, v13 +; GFX10-NEXT: v_lshlrev_b64 v[6:7], v10, v[6:7] +; GFX10-NEXT: v_min_u32_e32 v9, v12, v9 +; GFX10-NEXT: v_min_u32_e32 v11, v13, v14 +; GFX10-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX10-NEXT: v_min_u32_e32 v6, 1, v6 +; GFX10-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1] +; GFX10-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3] +; GFX10-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX10-NEXT: v_or_b32_e32 v5, v7, v6 +; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v11 ; GFX10-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX10-NEXT: v_min_u32_e32 v10, v12, v10 -; GFX10-NEXT: v_min_u32_e32 v11, v14, v13 -; GFX10-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3] -; GFX10-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX10-NEXT: v_lshlrev_b64 v[4:5], v10, v[4:5] -; GFX10-NEXT: v_lshlrev_b64 v[6:7], v11, v[6:7] ; GFX10-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX10-NEXT: v_min_u32_e32 v1, 1, v4 -; GFX10-NEXT: v_min_u32_e32 v4, 1, v6 +; GFX10-NEXT: v_sub_nc_u32_e32 v7, 32, v10 +; GFX10-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX10-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v4 ; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v8 -; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v10 -; GFX10-NEXT: v_or_b32_e32 v1, v5, v1 -; GFX10-NEXT: v_or_b32_e32 v4, v7, v4 -; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX10-NEXT: v_cvt_f32_i32_e32 v4, v5 +; GFX10-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX10-NEXT: v_sub_nc_u32_e32 v5, 32, v9 -; GFX10-NEXT: v_sub_nc_u32_e32 v7, 32, v11 -; GFX10-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX10-NEXT: v_cvt_f32_i32_e32 v4, v4 -; GFX10-NEXT: v_ldexp_f32 v0, v0, v3 -; GFX10-NEXT: v_ldexp_f32 v2, v2, v5 -; GFX10-NEXT: v_ldexp_f32 v1, v1, v6 +; GFX10-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX10-NEXT: v_ldexp_f32 v1, v1, v3 ; GFX10-NEXT: v_ldexp_f32 v3, v4, v7 -; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 +; GFX10-NEXT: v_ldexp_f32 v0, v0, v5 +; GFX10-NEXT: v_ldexp_f32 v2, v2, v6 ; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_sitofp_v4i64_to_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_xor_b32_e32 v8, v0, v1 -; GFX11-NEXT: v_cls_i32_e32 v9, v1 -; GFX11-NEXT: v_cls_i32_e32 v10, v3 -; GFX11-NEXT: v_xor_b32_e32 v11, v2, v3 -; GFX11-NEXT: v_xor_b32_e32 v13, v4, v5 +; GFX11-NEXT: v_xor_b32_e32 v8, v4, v5 +; GFX11-NEXT: v_cls_i32_e32 v9, v5 +; GFX11-NEXT: v_xor_b32_e32 v11, v6, v7 +; GFX11-NEXT: v_xor_b32_e32 v13, v0, v1 +; GFX11-NEXT: v_xor_b32_e32 v14, v2, v3 ; GFX11-NEXT: v_ashrrev_i32_e32 v8, 31, v8 ; GFX11-NEXT: v_add_nc_u32_e32 v9, -1, v9 -; GFX11-NEXT: v_xor_b32_e32 v15, v6, v7 -; GFX11-NEXT: v_cls_i32_e32 v12, v5 -; GFX11-NEXT: v_cls_i32_e32 v14, v7 -; GFX11-NEXT: v_add_nc_u32_e32 v8, 32, v8 +; GFX11-NEXT: v_cls_i32_e32 v10, v7 +; GFX11-NEXT: v_cls_i32_e32 v12, v1 ; GFX11-NEXT: v_ashrrev_i32_e32 v11, 31, v11 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_add_nc_u32_e32 v8, 32, v8 +; GFX11-NEXT: v_ashrrev_i32_e32 v14, 31, v14 +; GFX11-NEXT: v_add_nc_u32_e32 v10, -1, v10 ; GFX11-NEXT: v_add_nc_u32_e32 v12, -1, v12 -; GFX11-NEXT: v_add_nc_u32_e32 v14, -1, v14 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_min_u32_e32 v8, v9, v8 -; GFX11-NEXT: v_add_nc_u32_e32 v9, -1, v10 -; GFX11-NEXT: v_ashrrev_i32_e32 v10, 31, v13 -; GFX11-NEXT: v_ashrrev_i32_e32 v13, 31, v15 ; GFX11-NEXT: v_add_nc_u32_e32 v11, 32, v11 -; GFX11-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] +; GFX11-NEXT: v_min_u32_e32 v8, v9, v8 +; GFX11-NEXT: v_ashrrev_i32_e32 v9, 31, v13 +; GFX11-NEXT: v_cls_i32_e32 v13, v3 +; GFX11-NEXT: v_add_nc_u32_e32 v14, 32, v14 +; GFX11-NEXT: v_min_u32_e32 v10, v10, v11 +; GFX11-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX11-NEXT: v_add_nc_u32_e32 v9, 32, v9 +; GFX11-NEXT: v_add_nc_u32_e32 v13, -1, v13 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_lshlrev_b64 v[6:7], v10, v[6:7] +; GFX11-NEXT: v_min_u32_e32 v9, v12, v9 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_min_u32_e32 v11, v13, v14 +; GFX11-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX11-NEXT: v_min_u32_e32 v6, 1, v6 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_add_nc_u32_e32 v10, 32, v10 -; GFX11-NEXT: v_add_nc_u32_e32 v13, 32, v13 +; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1] +; GFX11-NEXT: v_lshlrev_b64 v[2:3], v11, v[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_min_u32_e32 v9, v9, v11 +; GFX11-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX11-NEXT: v_or_b32_e32 v5, v7, v6 +; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v11 ; GFX11-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_min_u32_e32 v10, v12, v10 -; GFX11-NEXT: v_min_u32_e32 v11, v14, v13 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3] -; GFX11-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_lshlrev_b64 v[4:5], v10, v[4:5] -; GFX11-NEXT: v_lshlrev_b64 v[6:7], v11, v[6:7] -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_min_u32_e32 v1, 1, v4 -; GFX11-NEXT: v_min_u32_e32 v4, 1, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_sub_nc_u32_e32 v7, 32, v10 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX11-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v4 ; GFX11-NEXT: v_sub_nc_u32_e32 v3, 32, v8 -; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v10 -; GFX11-NEXT: v_or_b32_e32 v1, v5, v1 -; GFX11-NEXT: v_or_b32_e32 v4, v7, v4 -; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX11-NEXT: v_cvt_f32_i32_e32 v4, v5 +; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0 ; GFX11-NEXT: v_sub_nc_u32_e32 v5, 32, v9 -; GFX11-NEXT: v_sub_nc_u32_e32 v7, 32, v11 -; GFX11-NEXT: v_cvt_f32_i32_e32 v1, v1 -; GFX11-NEXT: v_cvt_f32_i32_e32 v4, v4 -; GFX11-NEXT: v_ldexp_f32 v0, v0, v3 -; GFX11-NEXT: v_ldexp_f32 v2, v2, v5 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_ldexp_f32 v1, v1, v6 +; GFX11-NEXT: v_cvt_f32_i32_e32 v2, v2 +; GFX11-NEXT: v_ldexp_f32 v1, v1, v3 ; GFX11-NEXT: v_ldexp_f32 v3, v4, v7 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_ldexp_f32 v0, v0, v5 +; GFX11-NEXT: v_ldexp_f32 v2, v2, v6 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 ; GFX11-NEXT: v_perm_b32 v1, v3, v1, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = sitofp <4 x i64> %x to <4 x bfloat> ret <4 x bfloat> %op @@ -23961,13 +23752,11 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) { ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX8-NEXT: v_cvt_f32_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX8-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_uitofp_v3i16_to_v3bf16: @@ -23975,11 +23764,10 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) { ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-NEXT: v_cvt_f32_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX9-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 ; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4 -; GFX9-NEXT: v_perm_b32 v1, v1, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_uitofp_v3i16_to_v3bf16: @@ -23987,29 +23775,10 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) { ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX10-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX10-NEXT: v_cvt_f32_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX10-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX10-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_uitofp_v3i16_to_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_cvt_f32_u32_e32 v3, v3 -; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = uitofp <3 x i16> %x to <3 x bfloat> ret <3 x bfloat> %op } @@ -24052,55 +23821,55 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) { ; GFX8-LABEL: v_uitofp_v4i16_to_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX8-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX8-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX8-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX8-NEXT: v_cvt_f32_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX8-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_alignbit_b32 v0, v3, v0, 16 ; GFX8-NEXT: v_alignbit_b32 v1, v2, v1, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_uitofp_v4i16_to_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX9-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX9-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX9-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-NEXT: v_cvt_f32_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX9-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v2, s4 -; GFX9-NEXT: v_perm_b32 v1, v1, v3, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_uitofp_v4i16_to_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_cvt_f32_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX10-NEXT: v_cvt_f32_u32_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 +; GFX10-NEXT: v_cvt_f32_u32_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX10-NEXT: v_cvt_f32_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX10-NEXT: v_cvt_f32_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 ; GFX10-NEXT: v_cvt_f32_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v3, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_uitofp_v4i16_to_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v0 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v1 +; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v0 ; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v1 ; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_cvt_f32_u32_e32 v3, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v0, v3, 0x7060302 +; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = uitofp <4 x i16> %x to <4 x bfloat> ret <4 x bfloat> %op @@ -24236,12 +24005,12 @@ define <3 x bfloat> @v_uitofp_v3i32_to_v3bf16(<3 x i32> %x) { ; GFX8-LABEL: v_uitofp_v3i32_to_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX8-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16 +; GFX8-NEXT: v_cvt_f32_u32_e32 v3, v1 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3 +; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_uitofp_v3i32_to_v3bf16: @@ -24298,22 +24067,22 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) { ; GFX8-LABEL: v_uitofp_v4i32_to_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_cvt_f32_u32_e32 v3, v3 ; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1 ; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX8-NEXT: v_cvt_f32_u32_e32 v3, v3 ; GFX8-NEXT: v_cvt_f32_u32_e32 v2, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX8-NEXT: v_alignbit_b32 v0, v1, v0, 16 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v3 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_uitofp_v4i32_to_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v3 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 ; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 @@ -24323,9 +24092,9 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) { ; GFX10-LABEL: v_uitofp_v4i32_to_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX10-NEXT: v_cvt_f32_u32_e32 v3, v3 ; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x7060302 @@ -24334,11 +24103,11 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) { ; GFX11-LABEL: v_uitofp_v4i32_to_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX11-NEXT: v_cvt_f32_u32_e32 v3, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] @@ -24653,65 +24422,65 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) { ; GFX8-LABEL: v_uitofp_v3i64_to_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_ffbh_u32_e32 v6, v1 +; GFX8-NEXT: v_ffbh_u32_e32 v6, v5 ; GFX8-NEXT: v_min_u32_e32 v6, 32, v6 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] -; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX8-NEXT: v_cvt_f32_u32_e32 v7, v0 -; GFX8-NEXT: v_ffbh_u32_e32 v0, v3 -; GFX8-NEXT: v_min_u32_e32 v8, 32, v0 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6 -; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5] +; GFX8-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX8-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX8-NEXT: v_cvt_f32_u32_e32 v4, v4 +; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v6 +; GFX8-NEXT: v_ldexp_f32 v6, v4, v5 +; GFX8-NEXT: v_ffbh_u32_e32 v4, v1 +; GFX8-NEXT: v_min_u32_e32 v7, 32, v4 +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v7, v[0:1] +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v6 +; GFX8-NEXT: v_min_u32_e32 v0, 1, v4 +; GFX8-NEXT: v_ffbh_u32_e32 v4, v3 +; GFX8-NEXT: v_min_u32_e32 v4, 32, v4 +; GFX8-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3] +; GFX8-NEXT: v_or_b32_e32 v0, v5, v0 +; GFX8-NEXT: v_min_u32_e32 v2, 1, v2 +; GFX8-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX8-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v8 -; GFX8-NEXT: v_ldexp_f32 v2, v7, v2 -; GFX8-NEXT: v_ldexp_f32 v3, v0, v1 -; GFX8-NEXT: v_ffbh_u32_e32 v0, v5 -; GFX8-NEXT: v_min_u32_e32 v6, 32, v0 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5] -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v3, v2, 16 -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6 -; GFX8-NEXT: v_ldexp_f32 v1, v1, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v4 +; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v7 +; GFX8-NEXT: v_ldexp_f32 v2, v2, v3 +; GFX8-NEXT: v_ldexp_f32 v0, v0, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_uitofp_v3i64_to_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_ffbh_u32_e32 v6, v5 +; GFX9-NEXT: v_min_u32_e32 v6, 32, v6 +; GFX9-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5] +; GFX9-NEXT: s_mov_b32 s4, 0x7060302 +; GFX9-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX9-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX9-NEXT: v_sub_u32_e32 v5, 32, v6 ; GFX9-NEXT: v_ffbh_u32_e32 v6, v1 +; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v4 ; GFX9-NEXT: v_min_u32_e32 v6, 32, v6 ; GFX9-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] -; GFX9-NEXT: v_sub_u32_e32 v6, 32, v6 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v7, v0 +; GFX9-NEXT: v_ldexp_f32 v4, v4, v5 +; GFX9-NEXT: v_or_b32_e32 v5, v1, v0 ; GFX9-NEXT: v_ffbh_u32_e32 v0, v3 -; GFX9-NEXT: v_min_u32_e32 v8, 32, v0 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] -; GFX9-NEXT: v_ldexp_f32 v2, v7, v6 -; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v0 -; GFX9-NEXT: v_ffbh_u32_e32 v0, v5 ; GFX9-NEXT: v_min_u32_e32 v7, 32, v0 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[4:5] -; GFX9-NEXT: v_sub_u32_e32 v6, 32, v8 +; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[2:3] +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v5 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v0 -; GFX9-NEXT: v_ldexp_f32 v3, v3, v6 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v3, v2, s4 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0 +; GFX9-NEXT: v_sub_u32_e32 v1, 32, v6 +; GFX9-NEXT: v_ldexp_f32 v1, v2, v1 ; GFX9-NEXT: v_sub_u32_e32 v2, 32, v7 -; GFX9-NEXT: v_ldexp_f32 v1, v1, v2 -; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 +; GFX9-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v4, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_uitofp_v3i64_to_v3bf16: @@ -24726,21 +24495,21 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) { ; GFX10-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] ; GFX10-NEXT: v_lshlrev_b64 v[2:3], v7, v[2:3] ; GFX10-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] -; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v6 +; GFX10-NEXT: v_sub_nc_u32_e32 v8, 32, v8 ; GFX10-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX10-NEXT: v_min_u32_e32 v2, 1, v2 ; GFX10-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX10-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX10-NEXT: v_or_b32_e32 v1, v3, v2 ; GFX10-NEXT: v_or_b32_e32 v2, v5, v4 -; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v7 -; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v8 +; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v6 +; GFX10-NEXT: v_sub_nc_u32_e32 v4, 32, v7 ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0 ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1 ; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX10-NEXT: v_ldexp_f32 v0, v0, v6 -; GFX10-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX10-NEXT: v_ldexp_f32 v2, v2, v4 +; GFX10-NEXT: v_ldexp_f32 v0, v0, v3 +; GFX10-NEXT: v_ldexp_f32 v1, v1, v4 +; GFX10-NEXT: v_ldexp_f32 v2, v2, v8 ; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 ; GFX10-NEXT: v_alignbit_b32 v1, s4, v2, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] @@ -24834,129 +24603,129 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) { ; GFX8-LABEL: v_uitofp_v4i64_to_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_ffbh_u32_e32 v8, v1 +; GFX8-NEXT: v_ffbh_u32_e32 v8, v5 ; GFX8-NEXT: v_min_u32_e32 v8, 32, v8 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX8-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX8-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX8-NEXT: v_cvt_f32_u32_e32 v9, v4 +; GFX8-NEXT: v_ffbh_u32_e32 v4, v7 +; GFX8-NEXT: v_min_u32_e32 v10, 32, v4 +; GFX8-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7] +; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v8 +; GFX8-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX8-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX8-NEXT: v_cvt_f32_u32_e32 v4, v4 +; GFX8-NEXT: v_ldexp_f32 v5, v9, v6 +; GFX8-NEXT: v_sub_u32_e32 v6, vcc, 32, v10 +; GFX8-NEXT: v_ldexp_f32 v4, v4, v6 +; GFX8-NEXT: v_ffbh_u32_e32 v6, v1 +; GFX8-NEXT: v_min_u32_e32 v6, 32, v6 +; GFX8-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1] +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 ; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX8-NEXT: v_cvt_f32_u32_e32 v9, v0 +; GFX8-NEXT: v_cvt_f32_u32_e32 v7, v0 ; GFX8-NEXT: v_ffbh_u32_e32 v0, v3 -; GFX8-NEXT: v_min_u32_e32 v10, 32, v0 -; GFX8-NEXT: v_lshlrev_b64 v[0:1], v10, v[2:3] -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v8 +; GFX8-NEXT: v_min_u32_e32 v8, 32, v0 +; GFX8-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] +; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v6 ; GFX8-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX8-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX8-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX8-NEXT: v_sub_u32_e32 v1, vcc, 32, v10 -; GFX8-NEXT: v_ldexp_f32 v3, v9, v2 -; GFX8-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX8-NEXT: v_ffbh_u32_e32 v1, v5 -; GFX8-NEXT: v_min_u32_e32 v8, 32, v1 -; GFX8-NEXT: v_lshlrev_b64 v[1:2], v8, v[4:5] +; GFX8-NEXT: v_ldexp_f32 v1, v7, v2 +; GFX8-NEXT: v_sub_u32_e32 v2, vcc, 32, v8 +; GFX8-NEXT: v_ldexp_f32 v0, v0, v2 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX8-NEXT: v_or_b32_e32 v1, v2, v1 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 -; GFX8-NEXT: v_cvt_f32_u32_e32 v3, v1 -; GFX8-NEXT: v_ffbh_u32_e32 v1, v7 -; GFX8-NEXT: v_min_u32_e32 v4, 32, v1 -; GFX8-NEXT: v_lshlrev_b64 v[1:2], v4, v[6:7] -; GFX8-NEXT: v_sub_u32_e32 v5, vcc, 32, v8 -; GFX8-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX8-NEXT: v_or_b32_e32 v1, v2, v1 -; GFX8-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX8-NEXT: v_ldexp_f32 v2, v3, v5 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, 32, v4 -; GFX8-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v1, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v4, v5, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_uitofp_v4i64_to_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_ffbh_u32_e32 v8, v1 +; GFX9-NEXT: v_ffbh_u32_e32 v8, v5 ; GFX9-NEXT: v_min_u32_e32 v8, 32, v8 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 +; GFX9-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX9-NEXT: v_sub_u32_e32 v8, 32, v8 +; GFX9-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX9-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX9-NEXT: v_cvt_f32_u32_e32 v9, v4 +; GFX9-NEXT: v_ffbh_u32_e32 v4, v7 +; GFX9-NEXT: v_min_u32_e32 v10, 32, v4 +; GFX9-NEXT: v_lshlrev_b64 v[4:5], v10, v[6:7] +; GFX9-NEXT: v_ffbh_u32_e32 v7, v1 +; GFX9-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX9-NEXT: v_or_b32_e32 v4, v5, v4 +; GFX9-NEXT: v_cvt_f32_u32_e32 v4, v4 +; GFX9-NEXT: v_min_u32_e32 v7, 32, v7 +; GFX9-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1] +; GFX9-NEXT: v_sub_u32_e32 v5, 32, v10 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX9-NEXT: v_or_b32_e32 v9, v1, v0 +; GFX9-NEXT: v_ldexp_f32 v4, v4, v5 +; GFX9-NEXT: v_or_b32_e32 v5, v1, v0 ; GFX9-NEXT: v_ffbh_u32_e32 v0, v3 -; GFX9-NEXT: v_min_u32_e32 v10, 32, v0 -; GFX9-NEXT: v_lshlrev_b64 v[0:1], v10, v[2:3] -; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v9 +; GFX9-NEXT: v_ldexp_f32 v6, v9, v8 +; GFX9-NEXT: v_min_u32_e32 v8, 32, v0 +; GFX9-NEXT: v_lshlrev_b64 v[0:1], v8, v[2:3] +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, v5 ; GFX9-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX9-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX9-NEXT: v_sub_u32_e32 v1, 32, v8 -; GFX9-NEXT: v_ldexp_f32 v3, v2, v1 -; GFX9-NEXT: v_sub_u32_e32 v1, 32, v10 -; GFX9-NEXT: v_ldexp_f32 v0, v0, v1 -; GFX9-NEXT: v_ffbh_u32_e32 v1, v5 -; GFX9-NEXT: v_min_u32_e32 v8, 32, v1 -; GFX9-NEXT: v_lshlrev_b64 v[1:2], v8, v[4:5] -; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 -; GFX9-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX9-NEXT: v_or_b32_e32 v3, v2, v1 -; GFX9-NEXT: v_ffbh_u32_e32 v1, v7 -; GFX9-NEXT: v_min_u32_e32 v4, 32, v1 -; GFX9-NEXT: v_lshlrev_b64 v[1:2], v4, v[6:7] -; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v3 -; GFX9-NEXT: v_min_u32_e32 v1, 1, v1 -; GFX9-NEXT: v_or_b32_e32 v1, v2, v1 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, v1 +; GFX9-NEXT: v_sub_u32_e32 v1, 32, v7 +; GFX9-NEXT: v_ldexp_f32 v1, v2, v1 ; GFX9-NEXT: v_sub_u32_e32 v2, 32, v8 -; GFX9-NEXT: v_ldexp_f32 v2, v3, v2 -; GFX9-NEXT: v_sub_u32_e32 v3, 32, v4 -; GFX9-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_ldexp_f32 v0, v0, v2 +; GFX9-NEXT: s_mov_b32 s4, 0x7060302 +; GFX9-NEXT: v_perm_b32 v0, v0, v1, s4 +; GFX9-NEXT: v_perm_b32 v1, v4, v6, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_uitofp_v4i64_to_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_ffbh_u32_e32 v8, v1 -; GFX10-NEXT: v_ffbh_u32_e32 v9, v3 -; GFX10-NEXT: v_ffbh_u32_e32 v10, v5 +; GFX10-NEXT: v_ffbh_u32_e32 v8, v5 +; GFX10-NEXT: v_ffbh_u32_e32 v9, v1 +; GFX10-NEXT: v_ffbh_u32_e32 v10, v3 ; GFX10-NEXT: v_ffbh_u32_e32 v11, v7 ; GFX10-NEXT: v_min_u32_e32 v8, 32, v8 ; GFX10-NEXT: v_min_u32_e32 v9, 32, v9 ; GFX10-NEXT: v_min_u32_e32 v10, 32, v10 ; GFX10-NEXT: v_min_u32_e32 v11, 32, v11 -; GFX10-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] -; GFX10-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3] -; GFX10-NEXT: v_lshlrev_b64 v[4:5], v10, v[4:5] +; GFX10-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX10-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1] +; GFX10-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3] ; GFX10-NEXT: v_lshlrev_b64 v[6:7], v11, v[6:7] ; GFX10-NEXT: v_sub_nc_u32_e32 v8, 32, v8 +; GFX10-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX10-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX10-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX10-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX10-NEXT: v_min_u32_e32 v6, 1, v6 +; GFX10-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX10-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX10-NEXT: v_or_b32_e32 v1, v3, v2 -; GFX10-NEXT: v_or_b32_e32 v2, v5, v4 -; GFX10-NEXT: v_or_b32_e32 v4, v7, v6 -; GFX10-NEXT: v_sub_nc_u32_e32 v3, 32, v9 +; GFX10-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX10-NEXT: v_or_b32_e32 v3, v7, v6 +; GFX10-NEXT: v_sub_nc_u32_e32 v1, 32, v11 +; GFX10-NEXT: v_cvt_f32_u32_e32 v4, v4 ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, v1 +; GFX10-NEXT: v_sub_nc_u32_e32 v5, 32, v9 ; GFX10-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX10-NEXT: v_sub_nc_u32_e32 v5, 32, v10 -; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v11 -; GFX10-NEXT: v_cvt_f32_u32_e32 v4, v4 -; GFX10-NEXT: v_ldexp_f32 v0, v0, v8 -; GFX10-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX10-NEXT: v_ldexp_f32 v2, v2, v5 -; GFX10-NEXT: v_ldexp_f32 v3, v4, v6 -; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x7060302 +; GFX10-NEXT: v_sub_nc_u32_e32 v6, 32, v10 +; GFX10-NEXT: v_cvt_f32_u32_e32 v3, v3 +; GFX10-NEXT: v_ldexp_f32 v4, v4, v8 +; GFX10-NEXT: v_ldexp_f32 v0, v0, v5 +; GFX10-NEXT: v_ldexp_f32 v2, v2, v6 +; GFX10-NEXT: v_ldexp_f32 v1, v3, v1 +; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_uitofp_v4i64_to_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_clz_i32_u32_e32 v8, v1 -; GFX11-NEXT: v_clz_i32_u32_e32 v9, v3 -; GFX11-NEXT: v_clz_i32_u32_e32 v10, v5 +; GFX11-NEXT: v_clz_i32_u32_e32 v8, v5 +; GFX11-NEXT: v_clz_i32_u32_e32 v9, v1 +; GFX11-NEXT: v_clz_i32_u32_e32 v10, v3 ; GFX11-NEXT: v_clz_i32_u32_e32 v11, v7 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_min_u32_e32 v8, 32, v8 @@ -24965,37 +24734,37 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) { ; GFX11-NEXT: v_min_u32_e32 v10, 32, v10 ; GFX11-NEXT: v_min_u32_e32 v11, 32, v11 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1] -; GFX11-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3] +; GFX11-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5] +; GFX11-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_lshlrev_b64 v[4:5], v10, v[4:5] +; GFX11-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3] ; GFX11-NEXT: v_lshlrev_b64 v[6:7], v11, v[6:7] ; GFX11-NEXT: v_sub_nc_u32_e32 v8, 32, v8 +; GFX11-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX11-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX11-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX11-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX11-NEXT: v_min_u32_e32 v6, 1, v6 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX11-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX11-NEXT: v_or_b32_e32 v1, v3, v2 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_or_b32_e32 v2, v5, v4 -; GFX11-NEXT: v_or_b32_e32 v4, v7, v6 -; GFX11-NEXT: v_sub_nc_u32_e32 v3, 32, v9 -; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0 -; GFX11-NEXT: v_cvt_f32_u32_e32 v1, v1 -; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2 -; GFX11-NEXT: v_sub_nc_u32_e32 v5, 32, v10 -; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v11 -; GFX11-NEXT: v_cvt_f32_u32_e32 v4, v4 -; GFX11-NEXT: v_ldexp_f32 v0, v0, v8 -; GFX11-NEXT: v_ldexp_f32 v1, v1, v3 -; GFX11-NEXT: v_ldexp_f32 v2, v2, v5 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_ldexp_f32 v3, v4, v6 -; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x7060302 +; GFX11-NEXT: v_or_b32_e32 v2, v3, v2 +; GFX11-NEXT: v_or_b32_e32 v3, v7, v6 +; GFX11-NEXT: v_sub_nc_u32_e32 v1, 32, v11 +; GFX11-NEXT: v_cvt_f32_u32_e32 v4, v4 +; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0 +; GFX11-NEXT: v_sub_nc_u32_e32 v5, 32, v9 +; GFX11-NEXT: v_cvt_f32_u32_e32 v2, v2 +; GFX11-NEXT: v_sub_nc_u32_e32 v6, 32, v10 +; GFX11-NEXT: v_cvt_f32_u32_e32 v3, v3 +; GFX11-NEXT: v_ldexp_f32 v4, v4, v8 +; GFX11-NEXT: v_ldexp_f32 v0, v0, v5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_ldexp_f32 v2, v2, v6 +; GFX11-NEXT: v_ldexp_f32 v1, v3, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 +; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = uitofp <4 x i64> %x to <4 x bfloat> ret <4 x bfloat> %op @@ -25672,18 +25441,14 @@ define <3 x bfloat> @v_select_v3bf16(i1 %cond, <3 x bfloat> %a, <3 x bfloat> %b) ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_mov_b32_e32 v1, v2 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_select_v3bf16: @@ -25692,51 +25457,25 @@ define <3 x bfloat> @v_select_v3bf16(i1 %cond, <3 x bfloat> %a, <3 x bfloat> %b) ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_select_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v4 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v4, v7, v0, vcc_lo -; GFX10-NEXT: v_perm_b32 v0, v3, v1, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v4, v2, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_select_v3bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo -; GFX11-NEXT: v_dual_cndmask_b32 v3, v6, v5 :: v_dual_cndmask_b32 v2, v4, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v7, v0, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v3, v1, 0x5040100 -; GFX11-NEXT: v_perm_b32 v1, v4, v2, 0x5040100 +; GFX11-NEXT: v_dual_cndmask_b32 v0, v3, v1 :: v_dual_cndmask_b32 v1, v4, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select i1 %cond, <3 x bfloat> %a, <3 x bfloat> %b ret <3 x bfloat> %op @@ -25794,18 +25533,18 @@ define <4 x bfloat> @v_select_v4bf16(i1 %cond, <4 x bfloat> %a, <4 x bfloat> %b) ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v4 +; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v5 +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_select_v4bf16: @@ -25814,51 +25553,25 @@ define <4 x bfloat> @v_select_v4bf16(i1 %cond, <4 x bfloat> %a, <4 x bfloat> %b) ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_select_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v4 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v4, v7, v0, vcc_lo -; GFX10-NEXT: v_perm_b32 v0, v3, v1, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v4, v2, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_select_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v3 -; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc_lo -; GFX11-NEXT: v_dual_cndmask_b32 v3, v6, v5 :: v_dual_cndmask_b32 v2, v4, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v4, v7, v0, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v3, v1, 0x5040100 -; GFX11-NEXT: v_perm_b32 v1, v4, v2, 0x5040100 +; GFX11-NEXT: v_dual_cndmask_b32 v0, v3, v1 :: v_dual_cndmask_b32 v1, v4, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select i1 %cond, <4 x bfloat> %a, <4 x bfloat> %b ret <4 x bfloat> %op @@ -25932,24 +25645,24 @@ define <6 x bfloat> @v_select_v6bf16(i1 %cond, <6 x bfloat> %a, <6 x bfloat> %b) ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v6 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v5 +; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v5 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc +; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v6 +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v7 +; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_select_v6bf16: @@ -25958,65 +25671,28 @@ define <6 x bfloat> @v_select_v6bf16(i1 %cond, <6 x bfloat> %a, <6 x bfloat> %b) ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_select_v6bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v5 -; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v6 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc_lo -; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v4 -; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v5, v10, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v4, v2, 0x5040100 -; GFX10-NEXT: v_perm_b32 v2, v5, v3, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_select_v6bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v1 ; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v8, 16, v5 -; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v1 :: v_dual_cndmask_b32 v3, v6, v3 -; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v4 -; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v2 -; GFX11-NEXT: v_dual_cndmask_b32 v2, v5, v2 :: v_dual_cndmask_b32 v5, v10, v9 -; GFX11-NEXT: v_dual_cndmask_b32 v1, v1, v7 :: v_dual_cndmask_b32 v4, v8, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX11-NEXT: v_perm_b32 v1, v4, v2, 0x5040100 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_perm_b32 v2, v5, v3, 0x5040100 +; GFX11-NEXT: v_dual_cndmask_b32 v0, v4, v1 :: v_dual_cndmask_b32 v1, v5, v2 +; GFX11-NEXT: v_cndmask_b32_e32 v2, v6, v3, vcc_lo ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select i1 %cond, <6 x bfloat> %a, <6 x bfloat> %b ret <6 x bfloat> %op @@ -26106,30 +25782,30 @@ define <8 x bfloat> @v_select_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b) ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v1, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v8 +; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v7 +; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v6 +; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v5 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v4, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v7 +; GFX8-NEXT: v_cndmask_b32_e32 v9, v10, v9, vcc +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v8 +; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v9 +; GFX8-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_select_v8bf16: @@ -26138,80 +25814,30 @@ define <8 x bfloat> @v_select_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b) ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v1, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc -; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v4, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_perm_b32 v3, v4, v3, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_select_v8bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v5 -; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v4 -; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v8 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, v10, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v5, v6, v2, vcc_lo -; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v7, v12, v11, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v6, v10, v9, vcc_lo -; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v2, v5, 0x5040100 -; GFX10-NEXT: v_perm_b32 v2, v6, v3, 0x5040100 -; GFX10-NEXT: v_perm_b32 v3, v7, v4, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v3, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v3, v8, v4, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_select_v8bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v5 ; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v11, 16, v4 -; GFX11-NEXT: v_lshrrev_b32_e32 v12, 16, v8 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_dual_cndmask_b32 v0, v5, v1 :: v_dual_cndmask_b32 v5, v6, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v10, v9, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v3 -; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v7 -; GFX11-NEXT: v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v4, v8, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_dual_cndmask_b32 v2, v6, v2 :: v_dual_cndmask_b32 v7, v12, v11 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v10, v9, vcc_lo -; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_perm_b32 v1, v2, v5, 0x5040100 -; GFX11-NEXT: v_perm_b32 v2, v6, v3, 0x5040100 -; GFX11-NEXT: v_perm_b32 v3, v7, v4, 0x5040100 +; GFX11-NEXT: v_dual_cndmask_b32 v0, v5, v1 :: v_dual_cndmask_b32 v1, v6, v2 +; GFX11-NEXT: v_dual_cndmask_b32 v2, v7, v3 :: v_dual_cndmask_b32 v3, v8, v4 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b ret <8 x bfloat> %op @@ -26373,54 +25999,54 @@ define <16 x bfloat> @v_select_v16bf16(i1 %cond, <16 x bfloat> %a, <16 x bfloat> ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v9, v1, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v8 +; GFX8-NEXT: v_lshrrev_b32_e32 v18, 16, v16 +; GFX8-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v7 +; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v15 +; GFX8-NEXT: v_cndmask_b32_e32 v16, v16, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v14 +; GFX8-NEXT: v_cndmask_b32_e32 v15, v15, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v6, v14, v6, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v13 +; GFX8-NEXT: v_cndmask_b32_e32 v14, v14, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v12 +; GFX8-NEXT: v_cndmask_b32_e32 v13, v13, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v4, v12, v4, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v11 +; GFX8-NEXT: v_cndmask_b32_e32 v12, v12, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v10 +; GFX8-NEXT: v_cndmask_b32_e32 v11, v11, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v9 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v10, v2, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v10 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v2, v11, v3, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v11 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v3, v12, v4, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v12 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v4, v13, v5, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v13 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v5, v14, v6, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v14 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v6, v15, v7, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v15 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v7, v16, v8, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v16 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v11 +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v12 +; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v13 +; GFX8-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v14 +; GFX8-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v15 +; GFX8-NEXT: v_cndmask_b32_e32 v17, v18, v17, vcc +; GFX8-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v16 +; GFX8-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v17 +; GFX8-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_select_v16bf16: @@ -26429,140 +26055,40 @@ define <16 x bfloat> @v_select_v16bf16(i1 %cond, <16 x bfloat> %a, <16 x bfloat> ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v1, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v10, v2, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v10 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v2, vcc -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v11, v3, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v11 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v9, v3, vcc -; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v12, v4, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v12 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc -; GFX9-NEXT: v_perm_b32 v3, v4, v3, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v13, v5, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v13 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc -; GFX9-NEXT: v_perm_b32 v4, v5, v4, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v14, v6, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v14 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v9, v6, vcc -; GFX9-NEXT: v_perm_b32 v5, v6, v5, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v15, v7, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v15 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc -; GFX9-NEXT: v_perm_b32 v6, v7, v6, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v16, v8, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v16 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc -; GFX9-NEXT: v_perm_b32 v7, v8, v7, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_select_v16bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v18, 16, v9 -; GFX10-NEXT: v_lshrrev_b32_e32 v19, 16, v10 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v9, v18, v17, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v10, v19, v0, vcc_lo -; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v18, 16, v11 -; GFX10-NEXT: v_perm_b32 v0, v9, v1, 0x5040100 -; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v4 -; GFX10-NEXT: v_perm_b32 v1, v10, v2, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v9, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, v10, v2, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e32 v2, v11, v3, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v3, v18, v17, vcc_lo -; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v12 -; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v5 -; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v13 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo -; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v12, v4, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v4, v10, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v9, v17, v11, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v10, v14, v6, vcc_lo -; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v14 -; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v7 -; GFX10-NEXT: v_lshrrev_b32_e32 v13, 16, v15 -; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v8 -; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v11, v6, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v7, v15, v7, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v11, v13, v12, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v8, v16, v8, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v12, v17, v14, vcc_lo -; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x5040100 -; GFX10-NEXT: v_perm_b32 v4, v9, v5, 0x5040100 -; GFX10-NEXT: v_perm_b32 v5, v6, v10, 0x5040100 -; GFX10-NEXT: v_perm_b32 v6, v11, v7, 0x5040100 -; GFX10-NEXT: v_perm_b32 v7, v12, v8, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v4, v13, v5, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v5, v14, v6, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v6, v15, v7, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v7, v16, v8, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_select_v16bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v9 -; GFX11-NEXT: v_lshrrev_b32_e32 v19, 16, v10 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v2 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v9, v18, v17, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v10, v19, v0, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v3 -; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v11 -; GFX11-NEXT: v_perm_b32 v0, v9, v1, 0x5040100 -; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v4 -; GFX11-NEXT: v_perm_b32 v1, v10, v2, 0x5040100 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_dual_cndmask_b32 v2, v11, v3 :: v_dual_cndmask_b32 v3, v18, v17 -; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v12 -; GFX11-NEXT: v_lshrrev_b32_e32 v11, 16, v5 -; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v13 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v13, v5, vcc_lo -; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100 -; GFX11-NEXT: v_dual_cndmask_b32 v3, v12, v4 :: v_dual_cndmask_b32 v4, v10, v9 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_dual_cndmask_b32 v9, v17, v11 :: v_dual_cndmask_b32 v10, v14, v6 -; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX11-NEXT: v_lshrrev_b32_e32 v11, 16, v14 -; GFX11-NEXT: v_lshrrev_b32_e32 v12, 16, v7 -; GFX11-NEXT: v_lshrrev_b32_e32 v13, 16, v15 -; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v8 -; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v16 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v11, v6, vcc_lo -; GFX11-NEXT: v_dual_cndmask_b32 v7, v15, v7 :: v_dual_cndmask_b32 v8, v16, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v13, v12, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) -; GFX11-NEXT: v_cndmask_b32_e32 v12, v17, v14, vcc_lo -; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100 -; GFX11-NEXT: v_perm_b32 v4, v9, v5, 0x5040100 -; GFX11-NEXT: v_perm_b32 v5, v6, v10, 0x5040100 -; GFX11-NEXT: v_perm_b32 v6, v11, v7, 0x5040100 -; GFX11-NEXT: v_perm_b32 v7, v12, v8, 0x5040100 +; GFX11-NEXT: v_dual_cndmask_b32 v0, v9, v1 :: v_dual_cndmask_b32 v1, v10, v2 +; GFX11-NEXT: v_dual_cndmask_b32 v2, v11, v3 :: v_dual_cndmask_b32 v3, v12, v4 +; GFX11-NEXT: v_dual_cndmask_b32 v4, v13, v5 :: v_dual_cndmask_b32 v5, v14, v6 +; GFX11-NEXT: v_dual_cndmask_b32 v6, v15, v7 :: v_dual_cndmask_b32 v7, v16, v8 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select i1 %cond, <16 x bfloat> %a, <16 x bfloat> %b ret <16 x bfloat> %op @@ -26980,106 +26506,106 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat> ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v17, v1, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v18, v2, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v18 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v17, v2, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v2, v19, v3, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v19 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v3, v20, v4, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v20 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v17, v4, vcc -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v4, v21, v5, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_lshrrev_b32_e32 v18, 16, v21 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v18, v5, vcc -; GFX8-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:4 -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v5, v22, v6, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v22 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v19, v6, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v6, v23, v7, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v23 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v7, v24, v8, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v24 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v19, v8, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v8, v25, v9, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v25 -; GFX8-NEXT: v_cndmask_b32_e32 v9, v19, v9, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX8-NEXT: v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v9, v26, v10, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v26 -; GFX8-NEXT: v_cndmask_b32_e32 v10, v19, v10, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX8-NEXT: v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v10, v27, v11, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v27 -; GFX8-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v11, v28, v12, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v28 -; GFX8-NEXT: v_cndmask_b32_e32 v12, v19, v12, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_or_b32_sdwa v11, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v12, v29, v13, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v29 -; GFX8-NEXT: v_cndmask_b32_e32 v13, v19, v13, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v13, v30, v14, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v30 -; GFX8-NEXT: v_cndmask_b32_e32 v14, v19, v14, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_or_b32_sdwa v13, v13, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshrrev_b32_e32 v31, 16, v14 +; GFX8-NEXT: v_lshrrev_b32_e32 v32, 16, v30 +; GFX8-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v13 +; GFX8-NEXT: v_lshrrev_b32_e32 v30, 16, v29 +; GFX8-NEXT: v_cndmask_b32_e32 v30, v30, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v12 +; GFX8-NEXT: v_lshrrev_b32_e32 v29, 16, v28 +; GFX8-NEXT: v_cndmask_b32_e32 v29, v29, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v11 +; GFX8-NEXT: v_lshrrev_b32_e32 v28, 16, v27 +; GFX8-NEXT: v_cndmask_b32_e32 v28, v28, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v10 +; GFX8-NEXT: v_lshrrev_b32_e32 v27, 16, v26 +; GFX8-NEXT: v_cndmask_b32_e32 v27, v27, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v26, 16, v25 +; GFX8-NEXT: v_cndmask_b32_e32 v26, v26, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v8 +; GFX8-NEXT: v_lshrrev_b32_e32 v25, 16, v24 +; GFX8-NEXT: v_cndmask_b32_e32 v25, v25, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v7 +; GFX8-NEXT: v_lshrrev_b32_e32 v24, 16, v23 +; GFX8-NEXT: v_cndmask_b32_e32 v24, v24, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v23, 16, v22 +; GFX8-NEXT: v_cndmask_b32_e32 v23, v23, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v22, 16, v21 +; GFX8-NEXT: v_cndmask_b32_e32 v31, v32, v31, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v22, v22, v0, vcc +; GFX8-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 +; GFX8-NEXT: v_lshrrev_b32_e32 v33, 16, v16 +; GFX8-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v21, 16, v20 ; GFX8-NEXT: s_waitcnt vmcnt(1) -; GFX8-NEXT: v_cndmask_b32_e32 v14, v17, v15, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v17 -; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_or_b32_sdwa v14, v14, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_cndmask_b32_e32 v16, v0, v16, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_cndmask_b32_e32 v33, v0, v33, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v15 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cndmask_b32_e32 v15, v18, v16, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v18 -; GFX8-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v16, 16, v16 -; GFX8-NEXT: v_or_b32_sdwa v15, v15, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_cndmask_b32_e32 v15, v32, v15, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v32, 16, v32 +; GFX8-NEXT: v_cndmask_b32_e32 v32, v32, v0, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v4 +; GFX8-NEXT: v_cndmask_b32_e32 v21, v21, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v20, 16, v19 +; GFX8-NEXT: v_cndmask_b32_e32 v20, v20, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v18 +; GFX8-NEXT: v_cndmask_b32_e32 v19, v19, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v18, 16, v17 +; GFX8-NEXT: v_cndmask_b32_e32 v0, v18, v0, vcc +; GFX8-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc +; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v19 +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v20 +; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v21 +; GFX8-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v22 +; GFX8-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v23 +; GFX8-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v24 +; GFX8-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v25 +; GFX8-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v26 +; GFX8-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v27 +; GFX8-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v28 +; GFX8-NEXT: v_or_b32_sdwa v10, v11, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v29 +; GFX8-NEXT: v_or_b32_sdwa v11, v12, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v30 +; GFX8-NEXT: v_or_b32_sdwa v12, v13, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v31 +; GFX8-NEXT: v_or_b32_sdwa v13, v14, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v32 +; GFX8-NEXT: v_or_b32_sdwa v14, v15, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v33 +; GFX8-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_select_v32bf16: @@ -27088,90 +26614,25 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat> ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v17, v1, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v18, v2, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v18 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v17, v2, vcc -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 +; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 +; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:4 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v19, v3, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v19 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc -; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v20, v4, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v20 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v17, v4, vcc -; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 -; GFX9-NEXT: v_perm_b32 v3, v4, v3, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v21, v5, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v21 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v18, v5, vcc -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:4 -; GFX9-NEXT: v_perm_b32 v4, v5, v4, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v22, v6, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v22 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v19, v6, vcc -; GFX9-NEXT: v_perm_b32 v5, v6, v5, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v23, v7, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v23 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v19, v7, vcc -; GFX9-NEXT: v_perm_b32 v6, v7, v6, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v24, v8, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v24 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v19, v8, vcc -; GFX9-NEXT: v_perm_b32 v7, v8, v7, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v8, v25, v9, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v25 -; GFX9-NEXT: v_cndmask_b32_e32 v9, v19, v9, vcc -; GFX9-NEXT: v_perm_b32 v8, v9, v8, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v9, v26, v10, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v26 -; GFX9-NEXT: v_cndmask_b32_e32 v10, v19, v10, vcc -; GFX9-NEXT: v_perm_b32 v9, v10, v9, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v10, v27, v11, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v27 -; GFX9-NEXT: v_cndmask_b32_e32 v11, v19, v11, vcc -; GFX9-NEXT: v_perm_b32 v10, v11, v10, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v11, v28, v12, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v28 -; GFX9-NEXT: v_cndmask_b32_e32 v12, v19, v12, vcc -; GFX9-NEXT: v_perm_b32 v11, v12, v11, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v12, v29, v13, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v29 -; GFX9-NEXT: v_cndmask_b32_e32 v13, v19, v13, vcc -; GFX9-NEXT: v_perm_b32 v12, v13, v12, s4 ; GFX9-NEXT: v_cndmask_b32_e32 v13, v30, v14, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v30 -; GFX9-NEXT: v_cndmask_b32_e32 v14, v19, v14, vcc -; GFX9-NEXT: v_perm_b32 v13, v14, v13, s4 ; GFX9-NEXT: s_waitcnt vmcnt(1) ; GFX9-NEXT: v_cndmask_b32_e32 v14, v17, v15, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17 -; GFX9-NEXT: v_cndmask_b32_e32 v15, v17, v15, vcc -; GFX9-NEXT: v_perm_b32 v14, v15, v14, s4 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_cndmask_b32_e32 v15, v18, v16, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16 -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v18 -; GFX9-NEXT: v_cndmask_b32_e32 v16, v17, v16, vcc -; GFX9-NEXT: v_perm_b32 v15, v16, v15, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_select_v32bf16: @@ -27181,89 +26642,25 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat> ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 ; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v33, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v34, 16, v17 -; GFX10-NEXT: v_lshrrev_b32_e32 v35, 16, v2 -; GFX10-NEXT: v_lshrrev_b32_e32 v36, 16, v18 -; GFX10-NEXT: v_lshrrev_b32_e32 v37, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v38, 16, v19 -; GFX10-NEXT: v_lshrrev_b32_e32 v39, 16, v4 -; GFX10-NEXT: v_lshrrev_b32_e32 v48, 16, v20 -; GFX10-NEXT: v_lshrrev_b32_e32 v49, 16, v5 -; GFX10-NEXT: v_lshrrev_b32_e32 v50, 16, v21 -; GFX10-NEXT: v_lshrrev_b32_e32 v51, 16, v6 -; GFX10-NEXT: v_lshrrev_b32_e32 v52, 16, v22 -; GFX10-NEXT: v_lshrrev_b32_e32 v53, 16, v7 -; GFX10-NEXT: v_lshrrev_b32_e32 v54, 16, v23 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX10-NEXT: v_lshrrev_b32_e32 v55, 16, v8 -; GFX10-NEXT: v_lshrrev_b32_e32 v64, 16, v24 -; GFX10-NEXT: v_lshrrev_b32_e32 v65, 16, v9 -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v25 -; GFX10-NEXT: v_lshrrev_b32_e32 v66, 16, v10 -; GFX10-NEXT: v_lshrrev_b32_e32 v67, 16, v26 -; GFX10-NEXT: v_lshrrev_b32_e32 v68, 16, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v27 -; GFX10-NEXT: v_cndmask_b32_e32 v33, v34, v33, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v35, v36, v35, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v37, v38, v37, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v4, v20, v4, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v39, v48, v39, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v5, v21, v5, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v49, v50, v49, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v6, v22, v6, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v22, v52, v51, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v7, v23, v7, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v23, v54, v53, vcc_lo -; GFX10-NEXT: v_lshrrev_b32_e32 v34, 16, v12 -; GFX10-NEXT: v_lshrrev_b32_e32 v18, 16, v28 -; GFX10-NEXT: v_lshrrev_b32_e32 v36, 16, v13 -; GFX10-NEXT: v_lshrrev_b32_e32 v19, 16, v29 -; GFX10-NEXT: v_lshrrev_b32_e32 v38, 16, v14 -; GFX10-NEXT: v_lshrrev_b32_e32 v20, 16, v30 -; GFX10-NEXT: v_lshrrev_b32_e32 v48, 16, v15 -; GFX10-NEXT: v_lshrrev_b32_e32 v21, 16, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v24, v64, v55, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v9, v25, v9, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v25, v0, v65, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v10, v26, v10, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v26, v67, v66, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v11, v27, v11, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v17, v17, v68, vcc_lo -; GFX10-NEXT: v_perm_b32 v0, v33, v1, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v35, v2, 0x5040100 -; GFX10-NEXT: v_perm_b32 v2, v37, v3, 0x5040100 -; GFX10-NEXT: v_perm_b32 v3, v39, v4, 0x5040100 -; GFX10-NEXT: v_perm_b32 v4, v49, v5, 0x5040100 -; GFX10-NEXT: v_perm_b32 v5, v22, v6, 0x5040100 -; GFX10-NEXT: v_perm_b32 v6, v23, v7, 0x5040100 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v28, v12, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v18, v18, v34, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v13, v29, v13, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v19, v19, v36, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v20, v20, v38, vcc_lo -; GFX10-NEXT: v_perm_b32 v7, v24, v8, 0x5040100 -; GFX10-NEXT: v_perm_b32 v8, v25, v9, 0x5040100 -; GFX10-NEXT: v_perm_b32 v9, v26, v10, 0x5040100 -; GFX10-NEXT: v_perm_b32 v10, v17, v11, 0x5040100 -; GFX10-NEXT: v_perm_b32 v11, v18, v12, 0x5040100 -; GFX10-NEXT: v_perm_b32 v12, v19, v13, 0x5040100 -; GFX10-NEXT: v_perm_b32 v13, v20, v14, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v17, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, v18, v2, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v2, v19, v3, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v3, v20, v4, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v4, v21, v5, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v5, v22, v6, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v6, v23, v7, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v7, v24, v8, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v8, v25, v9, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v9, v26, v10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v10, v27, v11, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v11, v28, v12, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v12, v29, v13, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v13, v30, v14, vcc_lo ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_lshrrev_b32_e32 v22, 16, v31 +; GFX10-NEXT: v_cndmask_b32_e32 v14, v31, v15, vcc_lo ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_lshrrev_b32_e32 v23, 16, v32 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v31, v15, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v16, v32, v16, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v17, v22, v48, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v21, v23, v21, vcc_lo -; GFX10-NEXT: v_perm_b32 v14, v17, v15, 0x5040100 -; GFX10-NEXT: v_perm_b32 v15, v21, v16, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v15, v32, v16, vcc_lo ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_select_v32bf16: @@ -27273,80 +26670,17 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat> ; GFX11-NEXT: scratch_load_b32 v31, off, s32 ; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4 ; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v33, 16, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v34, 16, v17 -; GFX11-NEXT: v_lshrrev_b32_e32 v35, 16, v2 -; GFX11-NEXT: v_lshrrev_b32_e32 v36, 16, v18 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_lshrrev_b32_e32 v69, 16, v11 -; GFX11-NEXT: v_lshrrev_b32_e32 v70, 16, v27 -; GFX11-NEXT: v_lshrrev_b32_e32 v71, 16, v12 -; GFX11-NEXT: v_lshrrev_b32_e32 v80, 16, v28 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v81, 16, v13 -; GFX11-NEXT: v_lshrrev_b32_e32 v82, 16, v29 -; GFX11-NEXT: v_lshrrev_b32_e32 v83, 16, v14 -; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v30 -; GFX11-NEXT: v_cndmask_b32_e32 v17, v34, v33, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v2, v18, v2, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v18, v36, v35, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v37, 16, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v19, v3, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v38, 16, v19 -; GFX11-NEXT: v_lshrrev_b32_e32 v39, 16, v4 -; GFX11-NEXT: v_lshrrev_b32_e32 v48, 16, v20 -; GFX11-NEXT: v_lshrrev_b32_e32 v49, 16, v5 -; GFX11-NEXT: v_lshrrev_b32_e32 v50, 16, v21 -; GFX11-NEXT: v_lshrrev_b32_e32 v51, 16, v6 -; GFX11-NEXT: v_lshrrev_b32_e32 v52, 16, v22 -; GFX11-NEXT: v_lshrrev_b32_e32 v53, 16, v7 -; GFX11-NEXT: v_lshrrev_b32_e32 v54, 16, v23 -; GFX11-NEXT: v_lshrrev_b32_e32 v55, 16, v8 -; GFX11-NEXT: v_lshrrev_b32_e32 v64, 16, v24 -; GFX11-NEXT: v_lshrrev_b32_e32 v65, 16, v9 -; GFX11-NEXT: v_lshrrev_b32_e32 v66, 16, v25 -; GFX11-NEXT: v_lshrrev_b32_e32 v67, 16, v10 -; GFX11-NEXT: v_lshrrev_b32_e32 v68, 16, v26 -; GFX11-NEXT: v_lshrrev_b32_e32 v84, 16, v15 -; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v16 -; GFX11-NEXT: v_dual_cndmask_b32 v10, v26, v10 :: v_dual_cndmask_b32 v11, v27, v11 -; GFX11-NEXT: v_dual_cndmask_b32 v27, v70, v69 :: v_dual_cndmask_b32 v12, v28, v12 -; GFX11-NEXT: v_dual_cndmask_b32 v28, v80, v71 :: v_dual_cndmask_b32 v13, v29, v13 -; GFX11-NEXT: v_cndmask_b32_e32 v29, v82, v81, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v14, v30, v14, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v30, v0, v83, vcc_lo -; GFX11-NEXT: v_perm_b32 v0, v17, v1, 0x5040100 -; GFX11-NEXT: v_perm_b32 v1, v18, v2, 0x5040100 -; GFX11-NEXT: v_dual_cndmask_b32 v19, v38, v37 :: v_dual_cndmask_b32 v4, v20, v4 -; GFX11-NEXT: v_dual_cndmask_b32 v20, v48, v39 :: v_dual_cndmask_b32 v5, v21, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v21, v50, v49, vcc_lo -; GFX11-NEXT: v_dual_cndmask_b32 v6, v22, v6 :: v_dual_cndmask_b32 v7, v23, v7 -; GFX11-NEXT: v_dual_cndmask_b32 v22, v52, v51 :: v_dual_cndmask_b32 v23, v54, v53 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v24, v8, vcc_lo -; GFX11-NEXT: v_dual_cndmask_b32 v24, v64, v55 :: v_dual_cndmask_b32 v9, v25, v9 -; GFX11-NEXT: v_dual_cndmask_b32 v25, v66, v65 :: v_dual_cndmask_b32 v26, v68, v67 -; GFX11-NEXT: v_perm_b32 v2, v19, v3, 0x5040100 -; GFX11-NEXT: v_perm_b32 v3, v20, v4, 0x5040100 -; GFX11-NEXT: v_perm_b32 v4, v21, v5, 0x5040100 -; GFX11-NEXT: v_perm_b32 v5, v22, v6, 0x5040100 -; GFX11-NEXT: v_perm_b32 v6, v23, v7, 0x5040100 -; GFX11-NEXT: v_perm_b32 v7, v24, v8, 0x5040100 -; GFX11-NEXT: v_perm_b32 v8, v25, v9, 0x5040100 -; GFX11-NEXT: v_perm_b32 v9, v26, v10, 0x5040100 -; GFX11-NEXT: v_perm_b32 v10, v27, v11, 0x5040100 -; GFX11-NEXT: v_perm_b32 v11, v28, v12, 0x5040100 -; GFX11-NEXT: v_perm_b32 v12, v29, v13, 0x5040100 -; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v31 +; GFX11-NEXT: v_dual_cndmask_b32 v0, v17, v1 :: v_dual_cndmask_b32 v1, v18, v2 +; GFX11-NEXT: v_dual_cndmask_b32 v2, v19, v3 :: v_dual_cndmask_b32 v3, v20, v4 +; GFX11-NEXT: v_dual_cndmask_b32 v4, v21, v5 :: v_dual_cndmask_b32 v5, v22, v6 +; GFX11-NEXT: v_dual_cndmask_b32 v6, v23, v7 :: v_dual_cndmask_b32 v7, v24, v8 +; GFX11-NEXT: v_dual_cndmask_b32 v8, v25, v9 :: v_dual_cndmask_b32 v9, v26, v10 +; GFX11-NEXT: v_dual_cndmask_b32 v10, v27, v11 :: v_dual_cndmask_b32 v11, v28, v12 +; GFX11-NEXT: v_dual_cndmask_b32 v12, v29, v13 :: v_dual_cndmask_b32 v13, v30, v14 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_lshrrev_b32_e32 v18, 16, v32 -; GFX11-NEXT: v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v16, v32, v16 -; GFX11-NEXT: v_perm_b32 v13, v30, v14, 0x5040100 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_cndmask_b32 v17, v17, v84 :: v_dual_cndmask_b32 v18, v18, v85 -; GFX11-NEXT: v_perm_b32 v14, v17, v15, 0x5040100 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v15, v18, v16, 0x5040100 +; GFX11-NEXT: v_dual_cndmask_b32 v14, v31, v15 :: v_dual_cndmask_b32 v15, v32, v16 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select i1 %cond, <32 x bfloat> %a, <32 x bfloat> %b ret <32 x bfloat> %op @@ -27413,10 +26747,17 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat> ; GFX8-NEXT: v_mov_b32_e32 v2, s0 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GFX8-NEXT: s_lshr_b32 s0, s1, 16 +; GFX8-NEXT: s_lshr_b32 s2, s3, 16 ; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_mov_b32_e32 v1, s3 -; GFX8-NEXT: v_mov_b32_e32 v2, s1 +; GFX8-NEXT: v_mov_b32_e32 v1, s2 +; GFX8-NEXT: v_mov_b32_e32 v2, s0 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc +; GFX8-NEXT: v_mov_b32_e32 v2, s3 +; GFX8-NEXT: v_mov_b32_e32 v3, s1 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX8-NEXT: v_readfirstlane_b32 s0, v0 ; GFX8-NEXT: v_readfirstlane_b32 s1, v1 @@ -27424,17 +26765,10 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat> ; ; GFX9-LABEL: s_select_v3bf16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_lshr_b32 s4, s0, 16 -; GFX9-NEXT: s_lshr_b32 s5, s2, 16 -; GFX9-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NEXT: v_mov_b32_e32 v2, s4 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GFX9-NEXT: v_mov_b32_e32 v1, s2 ; GFX9-NEXT: v_mov_b32_e32 v2, s0 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX9-NEXT: s_mov_b32 s0, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v0, v1, s0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc ; GFX9-NEXT: v_mov_b32_e32 v1, s3 ; GFX9-NEXT: v_mov_b32_e32 v2, s1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc @@ -27445,39 +26779,28 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat> ; ; GFX10-LABEL: s_select_v3bf16: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_lshr_b32 s4, s0, 16 -; GFX10-NEXT: v_mov_b32_e32 v2, s0 -; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: v_mov_b32_e32 v1, s0 +; GFX10-NEXT: v_mov_b32_e32 v2, s1 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 -; GFX10-NEXT: v_mov_b32_e32 v0, s1 -; GFX10-NEXT: s_lshr_b32 s5, s2, 16 -; GFX10-NEXT: v_cndmask_b32_e32 v2, s2, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, s5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v0, s3, v0, vcc_lo -; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x5040100 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX10-NEXT: v_readfirstlane_b32 s0, v1 -; GFX10-NEXT: v_readfirstlane_b32 s1, v0 +; GFX10-NEXT: v_cndmask_b32_e32 v0, s2, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, s3, v2, vcc_lo +; GFX10-NEXT: v_readfirstlane_b32 s0, v0 +; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX10-NEXT: v_readfirstlane_b32 s1, v1 ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: s_select_v3bf16: ; GFX11: ; %bb.0: -; GFX11-NEXT: s_lshr_b32 s4, s0, 16 +; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 -; GFX11-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s0 -; GFX11-NEXT: v_mov_b32_e32 v0, s1 -; GFX11-NEXT: s_lshr_b32 s5, s2, 16 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: v_cndmask_b32_e32 v1, s5, v1, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e32 v2, s2, v2, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v0, s3, v0, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x5040100 -; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e32 v0, s2, v1, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v1, s3, v2, vcc_lo ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_readfirstlane_b32 s0, v1 -; GFX11-NEXT: v_readfirstlane_b32 s1, v0 +; GFX11-NEXT: v_readfirstlane_b32 s0, v0 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_readfirstlane_b32 s1, v1 ; GFX11-NEXT: ; return to shader part epilog %cond = icmp eq i32 %c, 0 %op = select i1 %cond, <3 x bfloat> %a, <3 x bfloat> %b @@ -27582,73 +26905,40 @@ define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat> ; GFX8-NEXT: v_readfirstlane_b32 s1, v0 ; GFX8-NEXT: ; return to shader part epilog ; -; GFX9-LABEL: s_select_v4bf16: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_lshr_b32 s4, s1, 16 -; GFX9-NEXT: s_lshr_b32 s5, s3, 16 -; GFX9-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NEXT: v_mov_b32_e32 v2, s4 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc +; GFX9-LABEL: s_select_v4bf16: +; GFX9: ; %bb.0: ; GFX9-NEXT: v_mov_b32_e32 v1, s3 ; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc +; GFX9-NEXT: v_mov_b32_e32 v1, s2 +; GFX9-NEXT: v_mov_b32_e32 v2, s0 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX9-NEXT: s_mov_b32 s1, 0x5040100 -; GFX9-NEXT: s_lshr_b32 s3, s0, 16 -; GFX9-NEXT: s_lshr_b32 s4, s2, 16 -; GFX9-NEXT: v_perm_b32 v0, v0, v1, s1 -; GFX9-NEXT: v_mov_b32_e32 v1, s4 -; GFX9-NEXT: v_mov_b32_e32 v2, s3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX9-NEXT: v_mov_b32_e32 v2, s2 -; GFX9-NEXT: v_mov_b32_e32 v3, s0 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s1 ; GFX9-NEXT: v_readfirstlane_b32 s0, v1 ; GFX9-NEXT: v_readfirstlane_b32 s1, v0 ; GFX9-NEXT: ; return to shader part epilog ; ; GFX10-LABEL: s_select_v4bf16: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_lshr_b32 s4, s1, 16 -; GFX10-NEXT: s_lshr_b32 s6, s0, 16 -; GFX10-NEXT: v_mov_b32_e32 v1, s4 -; GFX10-NEXT: v_mov_b32_e32 v2, s6 +; GFX10-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-NEXT: v_mov_b32_e32 v2, s0 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 -; GFX10-NEXT: v_mov_b32_e32 v0, s0 -; GFX10-NEXT: v_mov_b32_e32 v3, s1 -; GFX10-NEXT: s_lshr_b32 s5, s3, 16 -; GFX10-NEXT: s_lshr_b32 s0, s2, 16 -; GFX10-NEXT: v_cndmask_b32_e32 v1, s5, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v2, s0, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v3, s3, v3, vcc_lo -; GFX10-NEXT: v_perm_b32 v0, v2, v0, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x5040100 -; GFX10-NEXT: v_readfirstlane_b32 s0, v0 -; GFX10-NEXT: v_readfirstlane_b32 s1, v1 +; GFX10-NEXT: v_cndmask_b32_e32 v0, s3, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v1, s2, v2, vcc_lo +; GFX10-NEXT: v_readfirstlane_b32 s1, v0 +; GFX10-NEXT: v_readfirstlane_b32 s0, v1 ; GFX10-NEXT: ; return to shader part epilog ; ; GFX11-LABEL: s_select_v4bf16: ; GFX11: ; %bb.0: -; GFX11-NEXT: s_lshr_b32 s4, s1, 16 -; GFX11-NEXT: s_lshr_b32 s6, s0, 16 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s6 +; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s0 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 -; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s1 -; GFX11-NEXT: s_lshr_b32 s5, s3, 16 -; GFX11-NEXT: s_lshr_b32 s0, s2, 16 -; GFX11-NEXT: v_cndmask_b32_e32 v1, s5, v1, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v2, s0, v2, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v0, s2, v0, vcc_lo -; GFX11-NEXT: v_cndmask_b32_e32 v3, s3, v3, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x5040100 -; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x5040100 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e32 v0, s3, v1, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v1, s2, v2, vcc_lo ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_readfirstlane_b32 s0, v0 -; GFX11-NEXT: v_readfirstlane_b32 s1, v1 +; GFX11-NEXT: v_readfirstlane_b32 s1, v0 +; GFX11-NEXT: v_readfirstlane_b32 s0, v1 ; GFX11-NEXT: ; return to shader part epilog %cond = icmp eq i32 %c, 0 %op = select i1 %cond, <4 x bfloat> %a, <4 x bfloat> %b @@ -27887,96 +27177,95 @@ define <4 x bfloat> @v_vselect_v4bf16(<4 x i1> %cond, <4 x bfloat> %a, <4 x bflo ; GFX8-LABEL: v_vselect_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX8-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX8-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v7 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 ; GFX8-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v9, v8, vcc +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v5, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v6 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 +; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v5, vcc ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v4, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 -; GFX8-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc -; GFX8-NEXT: v_and_b32_e32 v3, 1, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v5, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v5 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v7 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v3 +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_vselect_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX9-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v5, vcc +; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 ; GFX9-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX9-NEXT: v_and_b32_e32 v2, 1, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v4, vcc ; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v6 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc ; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v5, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v5 -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v7 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 +; GFX9-NEXT: v_perm_b32 v1, v3, v2, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_vselect_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_and_b32_e32 v2, 1, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX10-NEXT: v_and_b32_e32 v1, 1, v1 -; GFX10-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX10-NEXT: v_and_b32_e32 v3, 1, v3 ; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v4 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 ; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v6 +; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v5, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX10-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v7 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v4, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v5 -; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v7 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 -; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v7, v5, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v6, v4, vcc_lo +; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc_lo ; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_vselect_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_and_b32_e32 v1, 1, v1 ; GFX11-NEXT: v_lshrrev_b32_e32 v8, 16, v4 ; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 +; GFX11-NEXT: v_dual_cndmask_b32 v2, v7, v5 :: v_dual_and_b32 v3, 1, v3 +; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5 +; GFX11-NEXT: v_lshrrev_b32_e32 v7, 16, v7 +; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_dual_cndmask_b32 v0, v6, v4 :: v_dual_and_b32 v3, 1, v3 +; GFX11-NEXT: v_dual_cndmask_b32 v0, v6, v4 :: v_dual_and_b32 v1, 1, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 -; GFX11-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v5 -; GFX11-NEXT: v_lshrrev_b32_e32 v6, 16, v7 ; GFX11-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 -; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v7, v5, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v6, v4, vcc_lo +; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v5, vcc_lo ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 ; GFX11-NEXT: s_setpc_b64 s[30:31] @@ -28062,172 +27351,170 @@ define <8 x bfloat> @v_vselect_v8bf16(<8 x i1> %cond, <8 x bfloat> %a, <8 x bflo ; GFX8-LABEL: v_vselect_v8bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX8-NEXT: v_and_b32_e32 v7, 1, v7 +; GFX8-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v11 +; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v15 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7 +; GFX8-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX8-NEXT: v_cndmask_b32_e32 v7, v17, v16, vcc +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 +; GFX8-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX8-NEXT: v_cndmask_b32_e32 v6, v15, v11, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v10 +; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v14 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5 +; GFX8-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v5, v15, v11, vcc +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4 +; GFX8-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX8-NEXT: v_cndmask_b32_e32 v4, v14, v10, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v13 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 ; GFX8-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v11, v10, vcc +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX8-NEXT: v_cndmask_b32_e32 v2, v13, v9, vcc +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v8 +; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v12 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 +; GFX8-NEXT: v_cndmask_b32_e32 v1, v10, v9, vcc ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v12, v8, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 -; GFX8-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v1, v12, v8, vcc -; GFX8-NEXT: v_and_b32_e32 v3, 1, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v1, v13, v9, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v9 -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v13 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 -; GFX8-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX8-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v2, v14, v10, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v10 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v14 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5 -; GFX8-NEXT: v_and_b32_e32 v6, 1, v6 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX8-NEXT: v_and_b32_e32 v7, 1, v7 -; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 -; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v3, v15, v11, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v11 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v15 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v3 +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v7 +; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_vselect_v8bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX9-NEXT: v_and_b32_e32 v7, 1, v7 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 +; GFX9-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX9-NEXT: v_cndmask_b32_e32 v6, v15, v11, vcc +; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v15 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7 +; GFX9-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v11, vcc +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4 +; GFX9-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX9-NEXT: v_cndmask_b32_e32 v4, v14, v10, vcc +; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v10 +; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v14 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5 +; GFX9-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX9-NEXT: v_cndmask_b32_e32 v5, v11, v10, vcc +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v13, v9, vcc +; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v13 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 ; GFX9-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v10, v9, vcc ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX9-NEXT: v_and_b32_e32 v2, 1, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v12, v8, vcc ; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v12 +; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v12 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v12, v8, vcc +; GFX9-NEXT: v_cndmask_b32_e32 v1, v9, v8, vcc ; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 -; GFX9-NEXT: v_and_b32_e32 v4, 1, v4 ; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v13, v9, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v9 -; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v13 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 -; GFX9-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4 -; GFX9-NEXT: v_and_b32_e32 v6, 1, v6 -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s4 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v14, v10, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v10 -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v14 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5 -; GFX9-NEXT: v_and_b32_e32 v7, 1, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 -; GFX9-NEXT: v_perm_b32 v2, v3, v2, s4 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v15, v11, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v11 -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v15 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc -; GFX9-NEXT: v_perm_b32 v3, v4, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v3, v2, s4 +; GFX9-NEXT: v_perm_b32 v2, v5, v4, s4 +; GFX9-NEXT: v_perm_b32 v3, v7, v6, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_vselect_v8bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX10-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX10-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX10-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX10-NEXT: v_lshrrev_b32_e32 v16, 16, v10 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 +; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v14 ; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX10-NEXT: v_and_b32_e32 v1, 1, v1 -; GFX10-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX10-NEXT: v_lshrrev_b32_e32 v16, 16, v8 -; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v12 -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 ; GFX10-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX10-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX10-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX10-NEXT: v_and_b32_e32 v6, 1, v6 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v8, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v9 -; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v13 +; GFX10-NEXT: v_cndmask_b32_e32 v6, v15, v11, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 ; GFX10-NEXT: v_and_b32_e32 v7, 1, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v17, v16, vcc_lo +; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v15 +; GFX10-NEXT: v_cndmask_b32_e32 v4, v14, v10, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 +; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v8 +; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v12 +; GFX10-NEXT: v_cndmask_b32_e32 v5, v17, v16, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 -; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 ; GFX10-NEXT: v_cndmask_b32_e32 v2, v13, v9, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX10-NEXT: v_lshrrev_b32_e32 v13, 16, v13 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v12, v8, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 +; GFX10-NEXT: v_cndmask_b32_e32 v1, v14, v10, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v14 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v12, v8, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 -; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v10 -; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v15 +; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v13, v9, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 ; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v14, v10, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 -; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v9, v8, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 +; GFX10-NEXT: v_cndmask_b32_e32 v7, v15, v11, vcc_lo ; GFX10-NEXT: v_perm_b32 v2, v5, v4, 0x5040100 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v15, v11, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v12, v10, vcc_lo ; GFX10-NEXT: v_perm_b32 v3, v7, v6, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_vselect_v8bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_lshrrev_b32_e32 v16, 16, v10 +; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v14 +; GFX11-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX11-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX11-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 +; GFX11-NEXT: v_dual_cndmask_b32 v6, v15, v11 :: v_dual_and_b32 v1, 1, v1 +; GFX11-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX11-NEXT: v_lshrrev_b32_e32 v15, 16, v15 +; GFX11-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 +; GFX11-NEXT: v_dual_cndmask_b32 v4, v14, v10 :: v_dual_and_b32 v3, 1, v3 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 +; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v8 +; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v12 ; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_and_b32_e32 v1, 1, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v16, 16, v8 -; GFX11-NEXT: v_lshrrev_b32_e32 v17, 16, v12 +; GFX11-NEXT: v_cndmask_b32_e32 v5, v17, v16, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 +; GFX11-NEXT: v_dual_cndmask_b32 v2, v13, v9 :: v_dual_and_b32 v7, 1, v7 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_dual_cndmask_b32 v0, v12, v8 :: v_dual_and_b32 v7, 1, v7 -; GFX11-NEXT: v_lshrrev_b32_e32 v8, 16, v9 -; GFX11-NEXT: v_lshrrev_b32_e32 v12, 16, v13 -; GFX11-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX11-NEXT: v_lshrrev_b32_e32 v13, 16, v13 +; GFX11-NEXT: v_cndmask_b32_e32 v0, v12, v8, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 -; GFX11-NEXT: v_dual_cndmask_b32 v1, v17, v16 :: v_dual_and_b32 v6, 1, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 -; GFX11-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v13, v9, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-NEXT: v_cndmask_b32_e32 v1, v14, v10, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 -; GFX11-NEXT: v_lshrrev_b32_e32 v9, 16, v14 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v12, v8, vcc_lo -; GFX11-NEXT: v_lshrrev_b32_e32 v12, 16, v15 -; GFX11-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX11-NEXT: v_lshrrev_b32_e32 v8, 16, v10 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX11-NEXT: v_cndmask_b32_e32 v3, v13, v9, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 ; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 -; GFX11-NEXT: v_dual_cndmask_b32 v4, v14, v10 :: v_dual_and_b32 v5, 1, v5 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 -; GFX11-NEXT: v_lshrrev_b32_e32 v10, 16, v11 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v9, v8, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 +; GFX11-NEXT: v_cndmask_b32_e32 v7, v15, v11, vcc_lo ; GFX11-NEXT: v_perm_b32 v2, v5, v4, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v15, v11, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v12, v10, vcc_lo -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_perm_b32 v3, v7, v6, 0x5040100 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select <8 x i1> %cond, <8 x bfloat> %a, <8 x bfloat> %b @@ -28469,244 +27756,259 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x ; GFX8-LABEL: v_vselect_v16bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill +; GFX8-NEXT: s_mov_b64 exec, s[4:5] ; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 ; GFX8-NEXT: v_and_b32_e32 v0, 1, v1 ; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0 -; GFX8-NEXT: v_cndmask_b32_e32 v0, v24, v16, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v24 -; GFX8-NEXT: v_cndmask_b32_e64 v1, v16, v1, s[4:5] -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_and_b32_e32 v1, 1, v2 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 -; GFX8-NEXT: v_and_b32_e32 v1, 1, v3 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v1 -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v17 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v25 -; GFX8-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v1, v25, v17, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_and_b32_e32 v2, 1, v4 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 -; GFX8-NEXT: v_and_b32_e32 v2, 1, v5 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2 -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v26 -; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, v3, s[4:5] -; GFX8-NEXT: v_cndmask_b32_e32 v2, v26, v18, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_and_b32_e32 v3, 1, v6 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 -; GFX8-NEXT: v_and_b32_e32 v3, 1, v7 -; GFX8-NEXT: buffer_load_dword v7, off, s[0:3], s32 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v3 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v2 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v3 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v4 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v5 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v6 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v7 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v8 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v9 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v10 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v0 +; GFX8-NEXT: buffer_load_dword v0, off, s[0:3], s32 +; GFX8-NEXT: v_writelane_b32 v31, s30, 0 +; GFX8-NEXT: v_and_b32_e32 v2, 1, v12 +; GFX8-NEXT: v_and_b32_e32 v3, 1, v13 +; GFX8-NEXT: v_writelane_b32 v31, s31, 1 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v2 +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v22 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v30 +; GFX8-NEXT: v_writelane_b32 v31, s34, 2 +; GFX8-NEXT: v_and_b32_e32 v1, 1, v11 +; GFX8-NEXT: v_and_b32_e32 v4, 1, v14 +; GFX8-NEXT: v_and_b32_e32 v5, 1, v15 +; GFX8-NEXT: v_cndmask_b32_e64 v6, v3, v2, s[28:29] +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v20 +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v28 +; GFX8-NEXT: v_writelane_b32 v31, s35, 3 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v23 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v4 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v5 +; GFX8-NEXT: v_cndmask_b32_e64 v10, v3, v2, s[20:21] +; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v21 +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v29 +; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v4, s[24:25] ; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v19 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v27 -; GFX8-NEXT: v_cndmask_b32_e64 v4, v5, v4, s[4:5] -; GFX8-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v27, v19, vcc -; GFX8-NEXT: v_and_b32_e32 v9, 1, v9 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8 -; GFX8-NEXT: v_and_b32_e32 v5, 1, v11 -; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v4, v28, v20, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v20 -; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v28 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v9 -; GFX8-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX8-NEXT: v_cndmask_b32_e32 v8, v11, v8, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10 -; GFX8-NEXT: v_or_b32_sdwa v4, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v8, v29, v21, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v21 -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v29 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5 -; GFX8-NEXT: v_and_b32_e32 v12, 1, v12 -; GFX8-NEXT: v_cndmask_b32_e32 v5, v10, v9, vcc -; GFX8-NEXT: v_and_b32_e32 v6, 1, v13 +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v27 +; GFX8-NEXT: v_cndmask_b32_e64 v3, v9, v4, s[16:17] +; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v24 +; GFX8-NEXT: v_cndmask_b32_e64 v7, v30, v22, s[26:27] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v27, v19, s[14:15] +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6 +; GFX8-NEXT: v_cndmask_b32_e64 v8, v29, v21, s[22:23] +; GFX8-NEXT: v_cndmask_b32_e64 v11, v28, v20, s[18:19] +; GFX8-NEXT: v_cndmask_b32_e64 v9, v26, v18, s[10:11] +; GFX8-NEXT: v_cndmask_b32_e64 v14, v25, v17, s[6:7] +; GFX8-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v10 ; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12 +; GFX8-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v4, v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: v_or_b32_sdwa v5, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e32 v8, v30, v22, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v22 -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v30 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 -; GFX8-NEXT: v_and_b32_e32 v14, 1, v14 -; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v9, vcc -; GFX8-NEXT: v_and_b32_e32 v15, 1, v15 -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v14 -; GFX8-NEXT: v_or_b32_sdwa v6, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v23 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cndmask_b32_e32 v8, v7, v23, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v15 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v0 +; GFX8-NEXT: v_cndmask_b32_e64 v12, v0, v23, s[30:31] +; GFX8-NEXT: v_cndmask_b32_e64 v13, v2, v1, s[34:35] +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v18 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v26 +; GFX8-NEXT: v_cndmask_b32_e64 v2, v1, v0, s[12:13] +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v17 +; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v25 +; GFX8-NEXT: v_cndmask_b32_e64 v1, v1, v0, s[8:9] +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v16 +; GFX8-NEXT: v_cndmask_b32_e64 v0, v15, v0, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e32 v15, v24, v16, vcc +; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v13 +; GFX8-NEXT: v_or_b32_sdwa v0, v15, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v1, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v7, v12, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_readlane_b32 s35, v31, 3 +; GFX8-NEXT: v_readlane_b32 s34, v31, 2 +; GFX8-NEXT: v_readlane_b32 s31, v31, 1 +; GFX8-NEXT: v_readlane_b32 s30, v31, 0 +; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 +; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload +; GFX8-NEXT: s_mov_b64 exec, s[4:5] +; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_vselect_v16bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v4 +; GFX9-NEXT: v_and_b32_e32 v4, 1, v14 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v4 +; GFX9-NEXT: v_and_b32_e32 v4, 1, v15 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v4 +; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 +; GFX9-NEXT: v_and_b32_e32 v12, 1, v12 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12 +; GFX9-NEXT: v_and_b32_e32 v12, 1, v13 +; GFX9-NEXT: v_and_b32_e32 v10, 1, v10 +; GFX9-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX9-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v12 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v10 +; GFX9-NEXT: v_and_b32_e32 v10, 1, v11 +; GFX9-NEXT: v_and_b32_e32 v8, 1, v8 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v6 +; GFX9-NEXT: v_and_b32_e32 v6, 1, v7 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v5 +; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v22 +; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v30 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v10 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v8 +; GFX9-NEXT: v_and_b32_e32 v8, 1, v9 +; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[4:5] +; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v21 +; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v29 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v8 +; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v5, s[8:9] +; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v20 +; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v28 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v6 +; GFX9-NEXT: v_cndmask_b32_e64 v9, v11, v9, s[12:13] +; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v19 +; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v27 +; GFX9-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX9-NEXT: v_cndmask_b32_e64 v11, v13, v11, s[16:17] +; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v23 +; GFX9-NEXT: v_cndmask_b32_e32 v6, v30, v22, vcc +; GFX9-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX9-NEXT: v_cndmask_b32_e64 v15, v26, v18, s[18:19] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v25, v17, vcc +; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX9-NEXT: v_cndmask_b32_e64 v8, v29, v21, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v10, v28, v20, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v27, v19, s[14:15] +; GFX9-NEXT: s_mov_b32 s4, 0x5040100 +; GFX9-NEXT: v_perm_b32 v5, v5, v8, s4 +; GFX9-NEXT: v_perm_b32 v6, v7, v6, s4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cndmask_b32_e64 v14, v4, v23, s[20:21] +; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4 +; GFX9-NEXT: v_cndmask_b32_e64 v13, v4, v13, s[22:23] +; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v18 +; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v26 +; GFX9-NEXT: v_cndmask_b32_e64 v4, v18, v4, s[24:25] +; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v25 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v18, v17, vcc ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 -; GFX9-NEXT: v_and_b32_e32 v0, 1, v1 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v24, v16, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v16 -; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v24 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v16, v1, s[4:5] -; GFX9-NEXT: s_mov_b32 s6, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s6 -; GFX9-NEXT: v_and_b32_e32 v1, 1, v2 +; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16 +; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v24 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 -; GFX9-NEXT: v_and_b32_e32 v1, 1, v3 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v1 -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v17 -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v25 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v25, v17, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[4:5] -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s6 -; GFX9-NEXT: v_and_b32_e32 v2, 1, v4 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 -; GFX9-NEXT: v_and_b32_e32 v2, 1, v5 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2 -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v18 -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v26 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v26, v18, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, v3, s[4:5] -; GFX9-NEXT: v_perm_b32 v2, v3, v2, s6 -; GFX9-NEXT: v_and_b32_e32 v3, 1, v6 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3 -; GFX9-NEXT: v_and_b32_e32 v3, 1, v7 -; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v3 -; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v19 -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v27 -; GFX9-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v27, v19, vcc -; GFX9-NEXT: v_cndmask_b32_e64 v4, v5, v4, s[4:5] -; GFX9-NEXT: v_and_b32_e32 v9, 1, v9 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8 -; GFX9-NEXT: v_and_b32_e32 v5, 1, v11 -; GFX9-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX9-NEXT: v_perm_b32 v3, v4, v3, s6 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v28, v20, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v20 -; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v28 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v9 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v11, v8, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10 -; GFX9-NEXT: v_and_b32_e32 v12, 1, v12 -; GFX9-NEXT: v_perm_b32 v4, v8, v4, s6 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v29, v21, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v21 -; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v29 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5 -; GFX9-NEXT: v_and_b32_e32 v6, 1, v13 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v10, v9, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12 -; GFX9-NEXT: v_and_b32_e32 v14, 1, v14 -; GFX9-NEXT: v_perm_b32 v5, v5, v8, s6 -; GFX9-NEXT: v_cndmask_b32_e32 v8, v30, v22, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v22 -; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v30 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6 -; GFX9-NEXT: v_and_b32_e32 v15, 1, v15 -; GFX9-NEXT: v_cndmask_b32_e32 v6, v10, v9, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v14 -; GFX9-NEXT: v_perm_b32 v6, v6, v8, s6 -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v23 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cndmask_b32_e32 v8, v7, v23, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v15 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc -; GFX9-NEXT: v_perm_b32 v7, v7, v8, s6 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v17, v16, vcc +; GFX9-NEXT: v_perm_b32 v0, v1, v0, s4 +; GFX9-NEXT: v_perm_b32 v1, v3, v2, s4 +; GFX9-NEXT: v_perm_b32 v2, v4, v15, s4 +; GFX9-NEXT: v_perm_b32 v3, v11, v12, s4 +; GFX9-NEXT: v_perm_b32 v4, v9, v10, s4 +; GFX9-NEXT: v_perm_b32 v7, v13, v14, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_vselect_v16bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX10-NEXT: v_and_b32_e32 v12, 1, v12 +; GFX10-NEXT: v_and_b32_e32 v13, 1, v13 +; GFX10-NEXT: v_and_b32_e32 v10, 1, v10 +; GFX10-NEXT: v_lshrrev_b32_e32 v33, 16, v22 +; GFX10-NEXT: v_lshrrev_b32_e32 v34, 16, v30 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v12 +; GFX10-NEXT: v_and_b32_e32 v11, 1, v11 +; GFX10-NEXT: v_and_b32_e32 v8, 1, v8 +; GFX10-NEXT: v_lshrrev_b32_e32 v35, 16, v21 +; GFX10-NEXT: v_lshrrev_b32_e32 v36, 16, v29 +; GFX10-NEXT: v_cndmask_b32_e32 v12, v30, v22, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13 +; GFX10-NEXT: v_and_b32_e32 v9, 1, v9 +; GFX10-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX10-NEXT: v_lshrrev_b32_e32 v37, 16, v20 +; GFX10-NEXT: v_lshrrev_b32_e32 v38, 16, v28 +; GFX10-NEXT: v_cndmask_b32_e32 v13, v34, v33, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v10 +; GFX10-NEXT: v_and_b32_e32 v4, 1, v4 ; GFX10-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX10-NEXT: v_lshrrev_b32_e32 v32, 16, v16 -; GFX10-NEXT: v_lshrrev_b32_e32 v33, 16, v24 -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 ; GFX10-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX10-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX10-NEXT: v_lshrrev_b32_e32 v34, 16, v17 -; GFX10-NEXT: v_lshrrev_b32_e32 v35, 16, v25 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v24, v16, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 +; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX10-NEXT: v_cndmask_b32_e32 v10, v29, v21, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v11 +; GFX10-NEXT: v_lshrrev_b32_e32 v51, 16, v17 +; GFX10-NEXT: v_lshrrev_b32_e32 v52, 16, v25 +; GFX10-NEXT: v_and_b32_e32 v1, 1, v1 ; GFX10-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX10-NEXT: v_and_b32_e32 v6, 1, v6 -; GFX10-NEXT: v_lshrrev_b32_e32 v36, 16, v18 -; GFX10-NEXT: v_lshrrev_b32_e32 v37, 16, v26 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v33, v32, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 +; GFX10-NEXT: v_cndmask_b32_e32 v11, v36, v35, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v8 +; GFX10-NEXT: v_lshrrev_b32_e32 v53, 16, v16 +; GFX10-NEXT: v_lshrrev_b32_e32 v54, 16, v24 ; GFX10-NEXT: v_and_b32_e32 v7, 1, v7 -; GFX10-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX10-NEXT: v_lshrrev_b32_e32 v38, 16, v19 -; GFX10-NEXT: v_lshrrev_b32_e32 v39, 16, v27 -; GFX10-NEXT: v_cndmask_b32_e32 v2, v25, v17, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 -; GFX10-NEXT: v_and_b32_e32 v9, 1, v9 -; GFX10-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX10-NEXT: v_lshrrev_b32_e32 v48, 16, v20 -; GFX10-NEXT: v_lshrrev_b32_e32 v49, 16, v28 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v35, v34, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 -; GFX10-NEXT: v_and_b32_e32 v11, 1, v11 -; GFX10-NEXT: v_and_b32_e32 v12, 1, v12 -; GFX10-NEXT: v_lshrrev_b32_e32 v50, 16, v21 -; GFX10-NEXT: v_lshrrev_b32_e32 v51, 16, v29 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v26, v18, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 -; GFX10-NEXT: v_and_b32_e32 v13, 1, v13 +; GFX10-NEXT: v_lshrrev_b32_e32 v49, 16, v18 +; GFX10-NEXT: v_cndmask_b32_e32 v8, v28, v20, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v9 +; GFX10-NEXT: v_lshrrev_b32_e32 v50, 16, v26 ; GFX10-NEXT: v_and_b32_e32 v14, 1, v14 -; GFX10-NEXT: v_lshrrev_b32_e32 v52, 16, v22 -; GFX10-NEXT: v_lshrrev_b32_e32 v53, 16, v30 -; GFX10-NEXT: v_cndmask_b32_e32 v5, v37, v36, vcc_lo +; GFX10-NEXT: v_lshrrev_b32_e32 v39, 16, v19 +; GFX10-NEXT: v_lshrrev_b32_e32 v48, 16, v27 +; GFX10-NEXT: v_cndmask_b32_e32 v9, v38, v37, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 ; GFX10-NEXT: v_and_b32_e32 v15, 1, v15 -; GFX10-NEXT: v_lshrrev_b32_e32 v54, 16, v23 -; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 +; GFX10-NEXT: v_lshrrev_b32_e32 v32, 16, v23 ; GFX10-NEXT: v_cndmask_b32_e32 v6, v27, v19, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 +; GFX10-NEXT: v_cndmask_b32_e32 v4, v26, v18, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 +; GFX10-NEXT: v_cndmask_b32_e32 v2, v25, v17, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v52, v51, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v24, v16, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 +; GFX10-NEXT: v_cndmask_b32_e32 v1, v54, v53, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 +; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX10-NEXT: v_cndmask_b32_e32 v5, v50, v49, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 +; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 ; GFX10-NEXT: v_perm_b32 v2, v5, v4, 0x5040100 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v39, v38, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v8 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v28, v20, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v9 -; GFX10-NEXT: v_cndmask_b32_e32 v9, v49, v48, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v10 +; GFX10-NEXT: v_cndmask_b32_e32 v7, v48, v39, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14 ; GFX10-NEXT: v_perm_b32 v4, v9, v8, 0x5040100 -; GFX10-NEXT: v_cndmask_b32_e32 v10, v29, v21, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v11, v51, v50, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v12 ; GFX10-NEXT: v_perm_b32 v5, v11, v10, 0x5040100 -; GFX10-NEXT: v_cndmask_b32_e32 v12, v30, v22, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13 -; GFX10-NEXT: v_cndmask_b32_e32 v13, v53, v52, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v31 ; GFX10-NEXT: v_cndmask_b32_e32 v14, v31, v23, vcc_lo ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v15 -; GFX10-NEXT: v_cndmask_b32_e32 v15, v3, v54, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v15, v3, v32, vcc_lo ; GFX10-NEXT: v_perm_b32 v3, v7, v6, 0x5040100 ; GFX10-NEXT: v_perm_b32 v6, v13, v12, 0x5040100 ; GFX10-NEXT: v_perm_b32 v7, v15, v14, 0x5040100 @@ -28716,78 +28018,79 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: v_lshrrev_b32_e32 v54, 16, v23 -; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX11-NEXT: v_lshrrev_b32_e32 v39, 16, v19 +; GFX11-NEXT: v_lshrrev_b32_e32 v48, 16, v27 +; GFX11-NEXT: v_and_b32_e32 v12, 1, v12 +; GFX11-NEXT: v_and_b32_e32 v13, 1, v13 +; GFX11-NEXT: v_lshrrev_b32_e32 v33, 16, v22 +; GFX11-NEXT: v_lshrrev_b32_e32 v34, 16, v30 +; GFX11-NEXT: v_lshrrev_b32_e32 v49, 16, v18 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v12 ; GFX11-NEXT: v_and_b32_e32 v1, 1, v1 -; GFX11-NEXT: v_lshrrev_b32_e32 v32, 16, v16 -; GFX11-NEXT: v_lshrrev_b32_e32 v33, 16, v24 -; GFX11-NEXT: v_lshrrev_b32_e32 v52, 16, v22 -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_and_b32_e32 v15, 1, v15 -; GFX11-NEXT: v_lshrrev_b32_e32 v53, 16, v30 +; GFX11-NEXT: v_lshrrev_b32_e32 v50, 16, v26 +; GFX11-NEXT: v_and_b32_e32 v10, 1, v10 +; GFX11-NEXT: v_dual_cndmask_b32 v12, v30, v22 :: v_dual_and_b32 v11, 1, v11 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13 +; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX11-NEXT: v_lshrrev_b32_e32 v35, 16, v21 +; GFX11-NEXT: v_lshrrev_b32_e32 v36, 16, v29 +; GFX11-NEXT: v_lshrrev_b32_e32 v53, 16, v16 +; GFX11-NEXT: v_cndmask_b32_e32 v13, v34, v33, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v10 +; GFX11-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX11-NEXT: v_lshrrev_b32_e32 v54, 16, v24 +; GFX11-NEXT: v_and_b32_e32 v8, 1, v8 +; GFX11-NEXT: v_and_b32_e32 v9, 1, v9 +; GFX11-NEXT: v_cndmask_b32_e32 v10, v29, v21, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v11 ; GFX11-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX11-NEXT: v_dual_cndmask_b32 v0, v24, v16 :: v_dual_and_b32 v3, 1, v3 -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 -; GFX11-NEXT: v_and_b32_e32 v14, 1, v14 -; GFX11-NEXT: v_lshrrev_b32_e32 v34, 16, v17 -; GFX11-NEXT: v_lshrrev_b32_e32 v35, 16, v25 -; GFX11-NEXT: v_lshrrev_b32_e32 v50, 16, v21 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v33, v32, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 -; GFX11-NEXT: v_and_b32_e32 v11, 1, v11 -; GFX11-NEXT: v_lshrrev_b32_e32 v51, 16, v29 -; GFX11-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX11-NEXT: v_lshrrev_b32_e32 v37, 16, v20 +; GFX11-NEXT: v_lshrrev_b32_e32 v38, 16, v28 +; GFX11-NEXT: v_lshrrev_b32_e32 v51, 16, v17 +; GFX11-NEXT: v_cndmask_b32_e32 v11, v36, v35, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v8 ; GFX11-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX11-NEXT: v_cndmask_b32_e32 v2, v25, v17, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 -; GFX11-NEXT: v_and_b32_e32 v12, 1, v12 -; GFX11-NEXT: v_lshrrev_b32_e32 v36, 16, v18 -; GFX11-NEXT: v_lshrrev_b32_e32 v37, 16, v26 -; GFX11-NEXT: v_lshrrev_b32_e32 v48, 16, v20 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v35, v34, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 -; GFX11-NEXT: v_and_b32_e32 v13, 1, v13 -; GFX11-NEXT: v_lshrrev_b32_e32 v49, 16, v28 +; GFX11-NEXT: v_lshrrev_b32_e32 v52, 16, v25 ; GFX11-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX11-NEXT: v_lshrrev_b32_e32 v32, 16, v23 +; GFX11-NEXT: v_cndmask_b32_e32 v8, v28, v20, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v9 +; GFX11-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX11-NEXT: v_and_b32_e32 v15, 1, v15 +; GFX11-NEXT: v_cndmask_b32_e32 v9, v38, v37, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 +; GFX11-NEXT: v_cndmask_b32_e32 v6, v27, v19, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 ; GFX11-NEXT: v_dual_cndmask_b32 v4, v26, v18 :: v_dual_and_b32 v7, 1, v7 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 +; GFX11-NEXT: v_cndmask_b32_e32 v2, v25, v17, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 +; GFX11-NEXT: v_cndmask_b32_e32 v3, v52, v51, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX11-NEXT: v_cndmask_b32_e32 v0, v24, v16, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 +; GFX11-NEXT: v_cndmask_b32_e32 v1, v54, v53, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 -; GFX11-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX11-NEXT: v_lshrrev_b32_e32 v38, 16, v19 -; GFX11-NEXT: v_lshrrev_b32_e32 v39, 16, v27 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3) ; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v37, v36, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 -; GFX11-NEXT: v_and_b32_e32 v9, 1, v9 +; GFX11-NEXT: v_cndmask_b32_e32 v5, v50, v49, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 ; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_perm_b32 v2, v5, v4, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v6, v27, v19, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 -; GFX11-NEXT: v_dual_cndmask_b32 v7, v39, v38 :: v_dual_and_b32 v8, 1, v8 -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v8 -; GFX11-NEXT: v_cndmask_b32_e32 v8, v28, v20, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v9 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v49, v48, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v10 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_cndmask_b32_e32 v7, v48, v39, vcc_lo ; GFX11-NEXT: v_perm_b32 v4, v9, v8, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v10, v29, v21, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v11 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v51, v50, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v12 ; GFX11-NEXT: v_perm_b32 v5, v11, v10, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v12, v30, v22, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v53, v52, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v31 +; GFX11-NEXT: v_and_b32_e32 v14, 1, v14 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14 ; GFX11-NEXT: v_cndmask_b32_e32 v14, v31, v23, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v15 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_cndmask_b32_e32 v15, v3, v54, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v15, v3, v32, vcc_lo ; GFX11-NEXT: v_perm_b32 v3, v7, v6, 0x5040100 ; GFX11-NEXT: v_perm_b32 v6, v13, v12, 0x5040100 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX11-NEXT: v_perm_b32 v7, v15, v14, 0x5040100 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select <16 x i1> %cond, <16 x bfloat> %a, <16 x bfloat> %b @@ -29252,171 +28555,235 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX8-NEXT: v_writelane_b32 v31, s30, 0 ; GFX8-NEXT: v_writelane_b32 v31, s31, 1 ; GFX8-NEXT: v_writelane_b32 v31, s34, 2 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_writelane_b32 v31, s35, 3 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v1 ; GFX8-NEXT: v_writelane_b32 v31, s36, 4 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v2 ; GFX8-NEXT: v_writelane_b32 v31, s37, 5 -; GFX8-NEXT: v_and_b32_e32 v21, 1, v21 -; GFX8-NEXT: v_and_b32_e32 v18, 1, v18 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v3 ; GFX8-NEXT: v_writelane_b32 v31, s38, 6 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v21 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v18 -; GFX8-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:68 -; GFX8-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:4 -; GFX8-NEXT: v_and_b32_e32 v17, 1, v17 -; GFX8-NEXT: v_and_b32_e32 v16, 1, v16 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v4 ; GFX8-NEXT: v_writelane_b32 v31, s39, 7 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v17 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v16 -; GFX8-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:72 -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:8 -; GFX8-NEXT: v_and_b32_e32 v15, 1, v15 -; GFX8-NEXT: v_and_b32_e32 v14, 1, v14 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v5 ; GFX8-NEXT: v_writelane_b32 v31, s40, 8 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v15 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[38:39], 1, v14 -; GFX8-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:76 -; GFX8-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:12 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v6 ; GFX8-NEXT: v_writelane_b32 v31, s41, 9 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v7 ; GFX8-NEXT: v_writelane_b32 v31, s42, 10 -; GFX8-NEXT: v_and_b32_e32 v13, 1, v13 -; GFX8-NEXT: v_and_b32_e32 v12, 1, v12 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v8 ; GFX8-NEXT: v_writelane_b32 v31, s43, 11 -; GFX8-NEXT: v_and_b32_e32 v20, 1, v20 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v13 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v12 -; GFX8-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:80 -; GFX8-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:16 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v20 -; GFX8-NEXT: buffer_load_ushort v20, off, s[0:3], s32 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v9 ; GFX8-NEXT: v_writelane_b32 v31, s44, 12 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v10 ; GFX8-NEXT: v_writelane_b32 v31, s45, 13 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v11 ; GFX8-NEXT: v_writelane_b32 v31, s46, 14 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v12 ; GFX8-NEXT: v_writelane_b32 v31, s47, 15 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v13 ; GFX8-NEXT: v_writelane_b32 v31, s48, 16 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v14 ; GFX8-NEXT: v_writelane_b32 v31, s49, 17 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v15 ; GFX8-NEXT: v_writelane_b32 v31, s50, 18 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v16 ; GFX8-NEXT: v_writelane_b32 v31, s51, 19 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v17 ; GFX8-NEXT: v_writelane_b32 v31, s52, 20 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[38:39], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v18 ; GFX8-NEXT: v_writelane_b32 v31, s53, 21 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v19 ; GFX8-NEXT: v_writelane_b32 v31, s54, 22 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v20 ; GFX8-NEXT: v_writelane_b32 v31, s55, 23 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v21 ; GFX8-NEXT: v_writelane_b32 v31, s56, 24 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[46:47], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v22 ; GFX8-NEXT: v_writelane_b32 v31, s57, 25 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[48:49], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v23 ; GFX8-NEXT: v_writelane_b32 v31, s58, 26 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[50:51], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v24 ; GFX8-NEXT: v_writelane_b32 v31, s59, 27 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[52:53], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v25 ; GFX8-NEXT: v_writelane_b32 v31, s60, 28 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[54:55], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v26 ; GFX8-NEXT: v_writelane_b32 v31, s61, 29 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[56:57], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v27 ; GFX8-NEXT: v_writelane_b32 v31, s62, 30 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v28 ; GFX8-NEXT: v_writelane_b32 v31, s63, 31 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v29 ; GFX8-NEXT: v_writelane_b32 v31, s64, 32 -; GFX8-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX8-NEXT: v_and_b32_e32 v7, 1, v7 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v0 +; GFX8-NEXT: v_and_b32_e32 v0, 1, v30 ; GFX8-NEXT: v_writelane_b32 v31, s65, 33 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[50:51], 1, v8 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[52:53], 1, v7 -; GFX8-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:84 -; GFX8-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:20 -; GFX8-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX8-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[64:65], 1, v0 +; GFX8-NEXT: buffer_load_ushort v0, off, s[0:3], s32 ; GFX8-NEXT: v_writelane_b32 v31, s66, 34 -; GFX8-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v2 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[64:65], 1, v1 -; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_writelane_b32 v31, s67, 35 -; GFX8-NEXT: v_and_b32_e32 v6, 1, v6 -; GFX8-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX8-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v3 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX8-NEXT: v_cmp_eq_u32_e64 s[66:67], 1, v0 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[54:55], 1, v6 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[56:57], 1, v5 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v4 -; GFX8-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:88 -; GFX8-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 -; GFX8-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX8-NEXT: v_and_b32_e32 v9, 1, v9 -; GFX8-NEXT: v_and_b32_e32 v11, 1, v11 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[46:47], 1, v10 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[48:49], 1, v9 -; GFX8-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:92 -; GFX8-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:28 -; GFX8-NEXT: v_and_b32_e32 v25, 1, v25 -; GFX8-NEXT: v_and_b32_e32 v24, 1, v24 -; GFX8-NEXT: v_and_b32_e32 v23, 1, v23 -; GFX8-NEXT: v_and_b32_e32 v22, 1, v22 -; GFX8-NEXT: v_and_b32_e32 v19, 1, v19 -; GFX8-NEXT: s_waitcnt vmcnt(14) -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v18 -; GFX8-NEXT: s_waitcnt vmcnt(13) -; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v21 -; GFX8-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[64:65] -; GFX8-NEXT: v_cndmask_b32_e64 v0, v18, v21, s[66:67] -; GFX8-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:36 -; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: s_waitcnt vmcnt(13) -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v16 -; GFX8-NEXT: s_waitcnt vmcnt(12) -; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v17 -; GFX8-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[60:61] -; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e64 v1, v16, v17, s[62:63] -; GFX8-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:40 +; GFX8-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 +; GFX8-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 +; GFX8-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 +; GFX8-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 +; GFX8-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 +; GFX8-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:12 +; GFX8-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 +; GFX8-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:16 +; GFX8-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:84 +; GFX8-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:20 +; GFX8-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:88 +; GFX8-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:24 +; GFX8-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92 +; GFX8-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:28 +; GFX8-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 +; GFX8-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:32 +; GFX8-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 +; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:36 +; GFX8-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:104 +; GFX8-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:40 +; GFX8-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:108 ; GFX8-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:44 -; GFX8-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56 -; GFX8-NEXT: s_waitcnt vmcnt(13) -; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v15 -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v14 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, v3, s[56:57] -; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e64 v2, v14, v15, s[58:59] -; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: s_waitcnt vmcnt(11) -; GFX8-NEXT: v_cndmask_b32_e64 v3, v12, v13, s[54:55] -; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v13 -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v11 -; GFX8-NEXT: s_waitcnt vmcnt(10) -; GFX8-NEXT: v_and_b32_e32 v11, 1, v20 -; GFX8-NEXT: v_cndmask_b32_e64 v4, v12, v4, s[52:53] -; GFX8-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:128 -; GFX8-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v25 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v24 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v23 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v22 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v19 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v11 -; GFX8-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:112 -; GFX8-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:108 -; GFX8-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:96 -; GFX8-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:32 -; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:104 -; GFX8-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:100 -; GFX8-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:48 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:120 -; GFX8-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:124 -; GFX8-NEXT: v_and_b32_e32 v26, 1, v26 -; GFX8-NEXT: v_and_b32_e32 v28, 1, v28 -; GFX8-NEXT: v_and_b32_e32 v27, 1, v27 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v26 -; GFX8-NEXT: v_and_b32_e32 v29, 1, v29 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v28 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v27 -; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v29 -; GFX8-NEXT: v_and_b32_e32 v30, 1, v30 -; GFX8-NEXT: s_waitcnt vmcnt(14) -; GFX8-NEXT: v_cndmask_b32_e64 v4, v7, v8, s[50:51] -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v8, s[48:49] -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_or_b32_sdwa v4, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v30 +; GFX8-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:112 +; GFX8-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:48 +; GFX8-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:116 +; GFX8-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:52 +; GFX8-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120 +; GFX8-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:56 +; GFX8-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:124 +; GFX8-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; GFX8-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:128 +; GFX8-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 +; GFX8-NEXT: s_waitcnt vmcnt(1) +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v29 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_lshrrev_b32_e32 v28, 16, v33 +; GFX8-NEXT: v_cndmask_b32_e64 v28, v34, v28, s[66:67] +; GFX8-NEXT: v_cndmask_b32_e64 v29, v29, v33, s[64:65] +; GFX8-NEXT: v_lshrrev_b32_e32 v33, 16, v32 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v30 +; GFX8-NEXT: v_cndmask_b32_e64 v33, v34, v33, s[62:63] +; GFX8-NEXT: v_cndmask_b32_e64 v30, v30, v32, s[60:61] +; GFX8-NEXT: v_lshrrev_b32_e32 v32, 16, v27 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v26 +; GFX8-NEXT: v_cndmask_b32_e64 v32, v34, v32, s[58:59] +; GFX8-NEXT: v_cndmask_b32_e64 v26, v26, v27, s[56:57] +; GFX8-NEXT: v_lshrrev_b32_e32 v27, 16, v25 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v24 +; GFX8-NEXT: v_cndmask_b32_e64 v27, v34, v27, s[54:55] +; GFX8-NEXT: v_cndmask_b32_e64 v24, v24, v25, s[52:53] +; GFX8-NEXT: v_lshrrev_b32_e32 v25, 16, v23 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v22 +; GFX8-NEXT: v_cndmask_b32_e64 v25, v34, v25, s[50:51] +; GFX8-NEXT: v_cndmask_b32_e64 v22, v22, v23, s[48:49] +; GFX8-NEXT: v_lshrrev_b32_e32 v23, 16, v21 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v20 +; GFX8-NEXT: v_cndmask_b32_e64 v23, v34, v23, s[46:47] +; GFX8-NEXT: v_cndmask_b32_e64 v20, v20, v21, s[44:45] +; GFX8-NEXT: v_lshrrev_b32_e32 v21, 16, v19 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v18 +; GFX8-NEXT: v_cndmask_b32_e64 v21, v34, v21, s[42:43] +; GFX8-NEXT: v_cndmask_b32_e64 v18, v18, v19, s[40:41] +; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v17 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v16 +; GFX8-NEXT: v_cndmask_b32_e64 v19, v34, v19, s[38:39] +; GFX8-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[36:37] +; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v15 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v14 +; GFX8-NEXT: v_cndmask_b32_e64 v17, v34, v17, s[34:35] +; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[30:31] +; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v13 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v12 +; GFX8-NEXT: v_cndmask_b32_e64 v15, v34, v15, s[28:29] +; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[26:27] +; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v11 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v10 +; GFX8-NEXT: v_cndmask_b32_e64 v13, v34, v13, s[24:25] +; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[22:23] +; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v9 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v8 +; GFX8-NEXT: v_cndmask_b32_e64 v11, v34, v11, s[20:21] +; GFX8-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[18:19] +; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v7 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v6 +; GFX8-NEXT: v_cndmask_b32_e64 v9, v34, v9, s[16:17] +; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[14:15] +; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v4 +; GFX8-NEXT: v_cndmask_b32_e64 v7, v34, v7, s[12:13] +; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11] +; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v2 +; GFX8-NEXT: v_cndmask_b32_e64 v5, v34, v5, s[8:9] +; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[6:7] +; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v1 +; GFX8-NEXT: v_lshrrev_b32_e32 v34, 16, v0 +; GFX8-NEXT: v_cndmask_b32_e64 v3, v34, v3, s[4:5] +; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v3 +; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v5 +; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v9 +; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v11 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v13 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v15 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v17 +; GFX8-NEXT: v_or_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v5, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v6, v12, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v7, v14, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v19 +; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v21 +; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v23 +; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v25 +; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v27 +; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v32 +; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v33 +; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v28 +; GFX8-NEXT: v_or_b32_sdwa v8, v16, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v9, v18, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v10, v20, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v11, v22, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v12, v24, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v14, v30, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX8-NEXT: v_or_b32_sdwa v15, v29, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: v_readlane_b32 s67, v31, 35 ; GFX8-NEXT: v_readlane_b32 s66, v31, 34 ; GFX8-NEXT: v_readlane_b32 s65, v31, 33 @@ -29427,18 +28794,6 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX8-NEXT: v_readlane_b32 s60, v31, 28 ; GFX8-NEXT: v_readlane_b32 s59, v31, 27 ; GFX8-NEXT: v_readlane_b32 s58, v31, 26 -; GFX8-NEXT: v_cndmask_b32_e64 v7, v5, v6, s[46:47] -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v6, s[44:45] -; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX8-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v10 -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v9 -; GFX8-NEXT: v_cndmask_b32_e64 v6, v9, v10, s[42:43] -; GFX8-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[40:41] -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7 -; GFX8-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: v_readlane_b32 s57, v31, 25 ; GFX8-NEXT: v_readlane_b32 s56, v31, 24 ; GFX8-NEXT: v_readlane_b32 s55, v31, 23 @@ -29457,43 +28812,6 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX8-NEXT: v_readlane_b32 s42, v31, 10 ; GFX8-NEXT: v_readlane_b32 s41, v31, 9 ; GFX8-NEXT: v_readlane_b32 s40, v31, 8 -; GFX8-NEXT: s_waitcnt vmcnt(6) -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v22 -; GFX8-NEXT: s_waitcnt vmcnt(5) -; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v23 -; GFX8-NEXT: v_cndmask_b32_e64 v8, v9, v8, s[36:37] -; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v18 -; GFX8-NEXT: s_waitcnt vmcnt(3) -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v25 -; GFX8-NEXT: v_cndmask_b32_e64 v7, v22, v23, s[38:39] -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX8-NEXT: v_cndmask_b32_e64 v9, v10, v9, s[30:31] -; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e64 v8, v25, v18, s[34:35] -; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v9 -; GFX8-NEXT: v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e64 v9, v24, v16, s[28:29] -; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v24 -; GFX8-NEXT: v_cndmask_b32_e64 v10, v16, v10, s[26:27] -; GFX8-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:52 -; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v10 -; GFX8-NEXT: v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_cndmask_b32_e64 v10, v11, v21, s[24:25] -; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v21 -; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v16, s[22:23] -; GFX8-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 -; GFX8-NEXT: v_lshlrev_b32_e32 v11, 16, v11 -; GFX8-NEXT: v_or_b32_sdwa v10, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: s_waitcnt vmcnt(4) -; GFX8-NEXT: v_cndmask_b32_e64 v11, v19, v20, s[20:21] -; GFX8-NEXT: v_lshrrev_b32_e32 v20, 16, v20 -; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v19 -; GFX8-NEXT: v_cndmask_b32_e64 v19, v19, v20, s[16:17] -; GFX8-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 -; GFX8-NEXT: v_lshlrev_b32_e32 v19, 16, v19 -; GFX8-NEXT: v_or_b32_sdwa v11, v11, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: v_readlane_b32 s39, v31, 7 ; GFX8-NEXT: v_readlane_b32 s38, v31, 6 ; GFX8-NEXT: v_readlane_b32 s37, v31, 5 @@ -29502,33 +28820,6 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX8-NEXT: v_readlane_b32 s34, v31, 2 ; GFX8-NEXT: v_readlane_b32 s31, v31, 1 ; GFX8-NEXT: v_readlane_b32 s30, v31, 0 -; GFX8-NEXT: s_waitcnt vmcnt(2) -; GFX8-NEXT: v_cndmask_b32_e64 v19, v12, v18, s[14:15] -; GFX8-NEXT: v_lshrrev_b32_e32 v18, 16, v18 -; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_cndmask_b32_e64 v12, v12, v18, s[12:13] -; GFX8-NEXT: v_cndmask_b32_e64 v18, v13, v17, s[10:11] -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v17 -; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v17, s[8:9] -; GFX8-NEXT: s_waitcnt vmcnt(1) -; GFX8-NEXT: v_cndmask_b32_e64 v17, v14, v16, s[6:7] -; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v16 -; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v16, s[4:5] -; GFX8-NEXT: v_lshlrev_b32_e32 v14, 16, v14 -; GFX8-NEXT: v_or_b32_sdwa v14, v17, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_cndmask_b32_e32 v16, v15, v20, vcc -; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v20 -; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v17, s[18:19] -; GFX8-NEXT: v_lshlrev_b32_e32 v12, 16, v12 -; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v13 -; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v15 -; GFX8-NEXT: v_or_b32_sdwa v12, v19, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_or_b32_sdwa v13, v18, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_or_b32_sdwa v15, v16, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; GFX8-NEXT: s_xor_saveexec_b64 s[4:5], -1 ; GFX8-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload ; GFX8-NEXT: s_mov_b64 exec, s[4:5] @@ -29544,169 +28835,223 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX9-NEXT: v_writelane_b32 v31, s30, 0 ; GFX9-NEXT: v_writelane_b32 v31, s31, 1 ; GFX9-NEXT: v_writelane_b32 v31, s34, 2 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_writelane_b32 v31, s35, 3 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v3 ; GFX9-NEXT: v_writelane_b32 v31, s36, 4 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v2 ; GFX9-NEXT: v_writelane_b32 v31, s37, 5 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v5 ; GFX9-NEXT: v_writelane_b32 v31, s38, 6 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v4 ; GFX9-NEXT: v_writelane_b32 v31, s39, 7 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v7 ; GFX9-NEXT: v_writelane_b32 v31, s40, 8 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v6 ; GFX9-NEXT: v_writelane_b32 v31, s41, 9 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v9 ; GFX9-NEXT: v_writelane_b32 v31, s42, 10 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v8 ; GFX9-NEXT: v_writelane_b32 v31, s43, 11 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v11 ; GFX9-NEXT: v_writelane_b32 v31, s44, 12 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v10 ; GFX9-NEXT: v_writelane_b32 v31, s45, 13 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v13 ; GFX9-NEXT: v_writelane_b32 v31, s46, 14 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v12 ; GFX9-NEXT: v_writelane_b32 v31, s47, 15 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v15 ; GFX9-NEXT: v_writelane_b32 v31, s48, 16 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v14 ; GFX9-NEXT: v_writelane_b32 v31, s49, 17 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v17 ; GFX9-NEXT: v_writelane_b32 v31, s50, 18 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v16 ; GFX9-NEXT: v_writelane_b32 v31, s51, 19 -; GFX9-NEXT: v_and_b32_e32 v21, 1, v21 -; GFX9-NEXT: v_and_b32_e32 v18, 1, v18 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[38:39], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v19 ; GFX9-NEXT: v_writelane_b32 v31, s52, 20 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v21 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v18 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:68 -; GFX9-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:4 -; GFX9-NEXT: v_and_b32_e32 v17, 1, v17 -; GFX9-NEXT: v_and_b32_e32 v16, 1, v16 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v18 ; GFX9-NEXT: v_writelane_b32 v31, s53, 21 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[30:31], 1, v17 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[34:35], 1, v16 -; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:72 -; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:8 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v21 ; GFX9-NEXT: v_writelane_b32 v31, s54, 22 -; GFX9-NEXT: v_and_b32_e32 v15, 1, v15 -; GFX9-NEXT: v_and_b32_e32 v14, 1, v14 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v20 ; GFX9-NEXT: v_writelane_b32 v31, s55, 23 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[36:37], 1, v15 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[38:39], 1, v14 -; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:76 -; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:12 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[46:47], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v23 ; GFX9-NEXT: v_writelane_b32 v31, s56, 24 -; GFX9-NEXT: v_and_b32_e32 v13, 1, v13 -; GFX9-NEXT: v_and_b32_e32 v12, 1, v12 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[48:49], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v22 ; GFX9-NEXT: v_writelane_b32 v31, s57, 25 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v13 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v12 -; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:80 -; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:16 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[50:51], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v25 ; GFX9-NEXT: v_writelane_b32 v31, s58, 26 -; GFX9-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX9-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[52:53], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v24 ; GFX9-NEXT: v_writelane_b32 v31, s59, 27 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[56:57], 1, v5 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v4 -; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:84 -; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:20 -; GFX9-NEXT: v_and_b32_e32 v20, 1, v20 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v20 -; GFX9-NEXT: buffer_load_ushort v20, off, s[0:3], s32 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[54:55], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v27 ; GFX9-NEXT: v_writelane_b32 v31, s60, 28 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[56:57], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v26 ; GFX9-NEXT: v_writelane_b32 v31, s61, 29 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[58:59], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v29 ; GFX9-NEXT: v_writelane_b32 v31, s62, 30 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v28 ; GFX9-NEXT: v_writelane_b32 v31, s63, 31 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v0 +; GFX9-NEXT: buffer_load_ushort v0, off, s[0:3], s32 ; GFX9-NEXT: v_writelane_b32 v31, s64, 32 ; GFX9-NEXT: v_writelane_b32 v31, s65, 33 ; GFX9-NEXT: v_writelane_b32 v31, s66, 34 -; GFX9-NEXT: v_and_b32_e32 v2, 1, v2 ; GFX9-NEXT: v_and_b32_e32 v1, 1, v1 -; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX9-NEXT: v_writelane_b32 v31, s67, 35 -; GFX9-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[62:63], 1, v2 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[64:65], 1, v1 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[64:65], 1, v0 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v30 ; GFX9-NEXT: v_cmp_eq_u32_e64 s[66:67], 1, v0 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[60:61], 1, v3 -; GFX9-NEXT: v_and_b32_e32 v6, 1, v6 -; GFX9-NEXT: v_and_b32_e32 v7, 1, v7 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[54:55], 1, v6 -; GFX9-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[52:53], 1, v7 -; GFX9-NEXT: v_and_b32_e32 v9, 1, v9 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[50:51], 1, v8 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[48:49], 1, v9 -; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:88 -; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:24 -; GFX9-NEXT: v_and_b32_e32 v24, 1, v24 -; GFX9-NEXT: v_and_b32_e32 v11, 1, v11 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[14:15], 1, v24 -; GFX9-NEXT: v_and_b32_e32 v23, 1, v23 -; GFX9-NEXT: v_and_b32_e32 v22, 1, v22 -; GFX9-NEXT: v_and_b32_e32 v19, 1, v19 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v11 -; GFX9-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:48 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v23 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v22 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v19 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[46:47], 1, v10 -; GFX9-NEXT: v_and_b32_e32 v26, 1, v26 -; GFX9-NEXT: v_and_b32_e32 v25, 1, v25 -; GFX9-NEXT: v_and_b32_e32 v28, 1, v28 -; GFX9-NEXT: v_and_b32_e32 v27, 1, v27 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v26 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 1, v25 -; GFX9-NEXT: v_and_b32_e32 v29, 1, v29 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v28 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 1, v27 -; GFX9-NEXT: v_and_b32_e32 v30, 1, v30 -; GFX9-NEXT: s_waitcnt vmcnt(13) -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v18 -; GFX9-NEXT: s_waitcnt vmcnt(12) -; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v21 -; GFX9-NEXT: v_cndmask_b32_e64 v0, v18, v21, s[66:67] -; GFX9-NEXT: v_cndmask_b32_e64 v1, v2, v1, s[64:65] -; GFX9-NEXT: s_mov_b32 s64, 0x5040100 -; GFX9-NEXT: v_perm_b32 v0, v1, v0, s64 -; GFX9-NEXT: s_waitcnt vmcnt(11) -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v16 -; GFX9-NEXT: s_waitcnt vmcnt(10) -; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v17 -; GFX9-NEXT: v_cndmask_b32_e64 v1, v16, v17, s[62:63] -; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[60:61] -; GFX9-NEXT: v_perm_b32 v1, v2, v1, s64 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:36 -; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:40 +; GFX9-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 +; GFX9-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 +; GFX9-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 +; GFX9-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 +; GFX9-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:76 +; GFX9-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:12 +; GFX9-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:80 +; GFX9-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:16 +; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:84 +; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:20 +; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:88 +; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:24 +; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:92 +; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:28 +; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:96 +; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:32 +; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:100 +; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:36 +; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:104 +; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:40 +; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:108 ; GFX9-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:44 -; GFX9-NEXT: s_waitcnt vmcnt(11) -; GFX9-NEXT: v_cndmask_b32_e64 v2, v14, v15, s[58:59] -; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v15 +; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:112 +; GFX9-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:48 +; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:116 +; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:52 +; GFX9-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:120 +; GFX9-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:56 +; GFX9-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:124 +; GFX9-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:60 +; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:128 +; GFX9-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:64 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cndmask_b32_e64 v29, v32, v33, s[66:67] +; GFX9-NEXT: v_lshrrev_b32_e32 v33, 16, v33 +; GFX9-NEXT: v_lshrrev_b32_e32 v32, 16, v32 +; GFX9-NEXT: v_cndmask_b32_e64 v32, v32, v33, s[64:65] +; GFX9-NEXT: v_cndmask_b32_e64 v33, v28, v30, s[62:63] +; GFX9-NEXT: v_lshrrev_b32_e32 v30, 16, v30 +; GFX9-NEXT: v_lshrrev_b32_e32 v28, 16, v28 +; GFX9-NEXT: v_cndmask_b32_e64 v28, v28, v30, s[60:61] +; GFX9-NEXT: v_cndmask_b32_e64 v30, v26, v27, s[58:59] +; GFX9-NEXT: v_lshrrev_b32_e32 v27, 16, v27 +; GFX9-NEXT: v_lshrrev_b32_e32 v26, 16, v26 +; GFX9-NEXT: v_cndmask_b32_e64 v26, v26, v27, s[56:57] +; GFX9-NEXT: v_cndmask_b32_e64 v27, v24, v25, s[54:55] +; GFX9-NEXT: v_lshrrev_b32_e32 v25, 16, v25 +; GFX9-NEXT: v_lshrrev_b32_e32 v24, 16, v24 +; GFX9-NEXT: v_cndmask_b32_e64 v24, v24, v25, s[52:53] +; GFX9-NEXT: v_cndmask_b32_e64 v25, v22, v23, s[50:51] +; GFX9-NEXT: v_lshrrev_b32_e32 v23, 16, v23 +; GFX9-NEXT: v_lshrrev_b32_e32 v22, 16, v22 +; GFX9-NEXT: v_cndmask_b32_e64 v22, v22, v23, s[48:49] +; GFX9-NEXT: v_cndmask_b32_e64 v23, v20, v21, s[46:47] +; GFX9-NEXT: v_lshrrev_b32_e32 v21, 16, v21 +; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v20 +; GFX9-NEXT: v_cndmask_b32_e64 v20, v20, v21, s[44:45] +; GFX9-NEXT: v_cndmask_b32_e64 v21, v18, v19, s[42:43] +; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v19 +; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v18 +; GFX9-NEXT: v_cndmask_b32_e64 v18, v18, v19, s[40:41] +; GFX9-NEXT: v_cndmask_b32_e64 v19, v16, v17, s[38:39] +; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17 +; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16 +; GFX9-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[36:37] +; GFX9-NEXT: v_cndmask_b32_e64 v17, v14, v15, s[34:35] +; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v15 ; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX9-NEXT: v_cndmask_b32_e64 v3, v14, v3, s[56:57] -; GFX9-NEXT: v_perm_b32 v2, v3, v2, s64 -; GFX9-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:124 -; GFX9-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:128 -; GFX9-NEXT: s_waitcnt vmcnt(11) -; GFX9-NEXT: v_cndmask_b32_e64 v3, v12, v13, s[54:55] +; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[30:31] +; GFX9-NEXT: v_cndmask_b32_e64 v15, v12, v13, s[28:29] ; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v13 ; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[52:53] -; GFX9-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:120 -; GFX9-NEXT: v_perm_b32 v3, v12, v3, s64 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v29 -; GFX9-NEXT: s_waitcnt vmcnt(10) -; GFX9-NEXT: v_cndmask_b32_e64 v12, v4, v5, s[50:51] +; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v13, s[26:27] +; GFX9-NEXT: v_cndmask_b32_e64 v13, v10, v11, s[24:25] +; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v11 +; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v10 +; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[22:23] +; GFX9-NEXT: v_cndmask_b32_e64 v11, v8, v9, s[20:21] +; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v9 +; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8 +; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[18:19] +; GFX9-NEXT: v_cndmask_b32_e64 v9, v6, v7, s[16:17] +; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7 +; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v6 +; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[14:15] +; GFX9-NEXT: v_cndmask_b32_e64 v7, v4, v5, s[12:13] ; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v5 ; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[48:49] -; GFX9-NEXT: v_perm_b32 v4, v4, v12, s64 -; GFX9-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:116 -; GFX9-NEXT: s_waitcnt vmcnt(10) -; GFX9-NEXT: v_and_b32_e32 v11, 1, v20 -; GFX9-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v11 -; GFX9-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:92 -; GFX9-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:28 -; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:112 -; GFX9-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:108 -; GFX9-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:104 -; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:100 -; GFX9-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:96 -; GFX9-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:32 -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v30 +; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11] +; GFX9-NEXT: v_cndmask_b32_e64 v5, v2, v3, s[8:9] +; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[6:7] +; GFX9-NEXT: v_cndmask_b32_e64 v3, v0, v1, s[4:5] +; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc +; GFX9-NEXT: s_mov_b32 s4, 0x5040100 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v2, v5, s4 +; GFX9-NEXT: v_perm_b32 v2, v4, v7, s4 +; GFX9-NEXT: v_perm_b32 v3, v6, v9, s4 +; GFX9-NEXT: v_perm_b32 v4, v8, v11, s4 +; GFX9-NEXT: v_perm_b32 v5, v10, v13, s4 +; GFX9-NEXT: v_perm_b32 v6, v12, v15, s4 +; GFX9-NEXT: v_perm_b32 v7, v14, v17, s4 +; GFX9-NEXT: v_perm_b32 v8, v16, v19, s4 +; GFX9-NEXT: v_perm_b32 v9, v18, v21, s4 +; GFX9-NEXT: v_perm_b32 v10, v20, v23, s4 +; GFX9-NEXT: v_perm_b32 v11, v22, v25, s4 +; GFX9-NEXT: v_perm_b32 v12, v24, v27, s4 +; GFX9-NEXT: v_perm_b32 v13, v26, v30, s4 +; GFX9-NEXT: v_perm_b32 v14, v28, v33, s4 +; GFX9-NEXT: v_perm_b32 v15, v32, v29, s4 ; GFX9-NEXT: v_readlane_b32 s67, v31, 35 ; GFX9-NEXT: v_readlane_b32 s66, v31, 34 ; GFX9-NEXT: v_readlane_b32 s65, v31, 33 +; GFX9-NEXT: v_readlane_b32 s64, v31, 32 ; GFX9-NEXT: v_readlane_b32 s63, v31, 31 ; GFX9-NEXT: v_readlane_b32 s62, v31, 30 ; GFX9-NEXT: v_readlane_b32 s61, v31, 29 @@ -29722,54 +29067,11 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX9-NEXT: v_readlane_b32 s51, v31, 19 ; GFX9-NEXT: v_readlane_b32 s50, v31, 18 ; GFX9-NEXT: v_readlane_b32 s49, v31, 17 -; GFX9-NEXT: s_waitcnt vmcnt(16) -; GFX9-NEXT: v_cndmask_b32_e64 v5, v6, v7, s[46:47] -; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v7 -; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[44:45] -; GFX9-NEXT: v_perm_b32 v5, v6, v5, s64 ; GFX9-NEXT: v_readlane_b32 s48, v31, 16 ; GFX9-NEXT: v_readlane_b32 s47, v31, 15 ; GFX9-NEXT: v_readlane_b32 s46, v31, 14 ; GFX9-NEXT: v_readlane_b32 s45, v31, 13 ; GFX9-NEXT: v_readlane_b32 s44, v31, 12 -; GFX9-NEXT: s_waitcnt vmcnt(6) -; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v9, s[42:43] -; GFX9-NEXT: v_lshrrev_b32_e32 v7, 16, v9 -; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v8, v7, s[40:41] -; GFX9-NEXT: v_perm_b32 v6, v7, v6, s64 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v22 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v23 -; GFX9-NEXT: v_cndmask_b32_e64 v7, v22, v23, s[38:39] -; GFX9-NEXT: v_cndmask_b32_e64 v8, v9, v8, s[36:37] -; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v18 -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v20 -; GFX9-NEXT: v_perm_b32 v7, v8, v7, s64 -; GFX9-NEXT: v_cndmask_b32_e64 v8, v20, v18, s[34:35] -; GFX9-NEXT: v_cndmask_b32_e64 v9, v17, v9, s[30:31] -; GFX9-NEXT: v_perm_b32 v8, v9, v8, s64 -; GFX9-NEXT: v_cndmask_b32_e64 v9, v11, v16, s[28:29] -; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16 -; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v11 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:52 -; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v16, s[26:27] -; GFX9-NEXT: v_perm_b32 v9, v11, v9, s64 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v10, v21, s[24:25] -; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v21 -; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v16, s[22:23] -; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 -; GFX9-NEXT: v_perm_b32 v10, v10, v11, s64 -; GFX9-NEXT: v_cndmask_b32_e64 v11, v19, v24, s[20:21] -; GFX9-NEXT: v_lshrrev_b32_e32 v20, 16, v24 -; GFX9-NEXT: v_lshrrev_b32_e32 v19, 16, v19 -; GFX9-NEXT: v_cndmask_b32_e64 v19, v19, v20, s[16:17] -; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:64 -; GFX9-NEXT: v_perm_b32 v11, v19, v11, s64 ; GFX9-NEXT: v_readlane_b32 s43, v31, 11 ; GFX9-NEXT: v_readlane_b32 s42, v31, 10 ; GFX9-NEXT: v_readlane_b32 s41, v31, 9 @@ -29782,31 +29084,6 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX9-NEXT: v_readlane_b32 s34, v31, 2 ; GFX9-NEXT: v_readlane_b32 s31, v31, 1 ; GFX9-NEXT: v_readlane_b32 s30, v31, 0 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: v_cndmask_b32_e64 v19, v12, v18, s[14:15] -; GFX9-NEXT: v_lshrrev_b32_e32 v18, 16, v18 -; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, v18, s[12:13] -; GFX9-NEXT: s_waitcnt vmcnt(2) -; GFX9-NEXT: v_cndmask_b32_e64 v18, v13, v17, s[10:11] -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v17 -; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v13 -; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v17, s[8:9] -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_cndmask_b32_e64 v17, v14, v16, s[6:7] -; GFX9-NEXT: v_lshrrev_b32_e32 v16, 16, v16 -; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX9-NEXT: v_cndmask_b32_e64 v14, v14, v16, s[4:5] -; GFX9-NEXT: v_perm_b32 v14, v14, v17, s64 -; GFX9-NEXT: v_perm_b32 v12, v12, v19, s64 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_cndmask_b32_e32 v16, v15, v20, vcc -; GFX9-NEXT: v_lshrrev_b32_e32 v17, 16, v20 -; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v15 -; GFX9-NEXT: v_cndmask_b32_e64 v15, v15, v17, s[18:19] -; GFX9-NEXT: v_perm_b32 v13, v13, v18, s64 -; GFX9-NEXT: v_perm_b32 v15, v15, v16, s64 -; GFX9-NEXT: v_readlane_b32 s64, v31, 32 ; GFX9-NEXT: s_xor_saveexec_b64 s[4:5], -1 ; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[4:5] @@ -29820,205 +29097,208 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX10-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s4 -; GFX10-NEXT: v_and_b32_e32 v3, 1, v3 -; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX10-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX10-NEXT: v_and_b32_e32 v1, 1, v1 -; GFX10-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 1, v3 -; GFX10-NEXT: v_and_b32_e32 v3, 1, v6 -; GFX10-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX10-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX10-NEXT: v_and_b32_e32 v12, 1, v12 -; GFX10-NEXT: v_and_b32_e32 v14, 1, v14 -; GFX10-NEXT: v_and_b32_e32 v16, 1, v16 -; GFX10-NEXT: s_clause 0x15 -; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:68 -; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:4 -; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:72 -; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:8 -; GFX10-NEXT: buffer_load_ushort v36, off, s[0:3], s32 -; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:76 -; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:12 -; GFX10-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:80 -; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:16 -; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:20 -; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:84 -; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:88 -; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:24 -; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:92 -; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:28 -; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:96 -; GFX10-NEXT: buffer_load_dword v64, off, s[0:3], s32 offset:32 -; GFX10-NEXT: buffer_load_dword v65, off, s[0:3], s32 offset:36 -; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:104 -; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:40 -; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:100 -; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:52 -; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 1, v0 -; GFX10-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:112 -; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 1, v2 -; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:48 -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 -; GFX10-NEXT: v_cmp_eq_u32_e64 s7, 1, v4 -; GFX10-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:120 -; GFX10-NEXT: v_cmp_eq_u32_e64 s8, 1, v3 -; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:56 -; GFX10-NEXT: v_cmp_eq_u32_e64 s9, 1, v8 -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:116 -; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:108 -; GFX10-NEXT: v_cmp_eq_u32_e64 s10, 1, v10 -; GFX10-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:124 -; GFX10-NEXT: v_cmp_eq_u32_e64 s11, 1, v12 -; GFX10-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 -; GFX10-NEXT: v_cmp_eq_u32_e64 s12, 1, v14 -; GFX10-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:128 -; GFX10-NEXT: v_cmp_eq_u32_e64 s13, 1, v16 -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 -; GFX10-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:44 -; GFX10-NEXT: v_writelane_b32 v31, s30, 0 +; GFX10-NEXT: v_and_b32_e32 v29, 1, v29 ; GFX10-NEXT: v_and_b32_e32 v30, 1, v30 ; GFX10-NEXT: v_and_b32_e32 v28, 1, v28 ; GFX10-NEXT: v_and_b32_e32 v26, 1, v26 ; GFX10-NEXT: v_and_b32_e32 v24, 1, v24 -; GFX10-NEXT: v_writelane_b32 v31, s31, 1 ; GFX10-NEXT: v_and_b32_e32 v22, 1, v22 ; GFX10-NEXT: v_and_b32_e32 v20, 1, v20 -; GFX10-NEXT: v_and_b32_e32 v17, 1, v17 -; GFX10-NEXT: v_and_b32_e32 v9, 1, v9 -; GFX10-NEXT: v_and_b32_e32 v7, 1, v7 -; GFX10-NEXT: v_writelane_b32 v31, s34, 2 -; GFX10-NEXT: v_and_b32_e32 v29, 1, v29 -; GFX10-NEXT: v_and_b32_e32 v27, 1, v27 -; GFX10-NEXT: v_and_b32_e32 v25, 1, v25 -; GFX10-NEXT: v_and_b32_e32 v23, 1, v23 -; GFX10-NEXT: v_and_b32_e32 v21, 1, v21 -; GFX10-NEXT: v_and_b32_e32 v19, 1, v19 ; GFX10-NEXT: v_and_b32_e32 v18, 1, v18 -; GFX10-NEXT: v_and_b32_e32 v15, 1, v15 -; GFX10-NEXT: v_and_b32_e32 v13, 1, v13 -; GFX10-NEXT: v_and_b32_e32 v11, 1, v11 +; GFX10-NEXT: v_and_b32_e32 v16, 1, v16 +; GFX10-NEXT: v_and_b32_e32 v14, 1, v14 +; GFX10-NEXT: v_and_b32_e32 v12, 1, v12 +; GFX10-NEXT: s_clause 0x14 +; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:60 +; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:124 +; GFX10-NEXT: buffer_load_ushort v34, off, s[0:3], s32 +; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:128 +; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:64 +; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:48 +; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:116 +; GFX10-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:52 +; GFX10-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:120 +; GFX10-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:56 +; GFX10-NEXT: buffer_load_dword v50, off, s[0:3], s32 offset:32 +; GFX10-NEXT: buffer_load_dword v51, off, s[0:3], s32 offset:100 +; GFX10-NEXT: buffer_load_dword v52, off, s[0:3], s32 offset:36 +; GFX10-NEXT: buffer_load_dword v53, off, s[0:3], s32 offset:104 +; GFX10-NEXT: buffer_load_dword v54, off, s[0:3], s32 offset:40 +; GFX10-NEXT: buffer_load_dword v55, off, s[0:3], s32 offset:108 +; GFX10-NEXT: buffer_load_dword v64, off, s[0:3], s32 offset:44 +; GFX10-NEXT: buffer_load_dword v65, off, s[0:3], s32 offset:112 +; GFX10-NEXT: buffer_load_dword v66, off, s[0:3], s32 offset:72 +; GFX10-NEXT: buffer_load_dword v67, off, s[0:3], s32 offset:76 +; GFX10-NEXT: buffer_load_dword v68, off, s[0:3], s32 offset:80 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v29 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: buffer_load_dword v29, off, s[0:3], s32 offset:92 +; GFX10-NEXT: buffer_load_dword v69, off, s[0:3], s32 offset:28 +; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 1, v30 +; GFX10-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:96 +; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 1, v28 +; GFX10-NEXT: buffer_load_dword v28, off, s[0:3], s32 offset:88 +; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 1, v26 +; GFX10-NEXT: v_cmp_eq_u32_e64 s7, 1, v24 +; GFX10-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:84 +; GFX10-NEXT: v_cmp_eq_u32_e64 s8, 1, v22 +; GFX10-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:20 +; GFX10-NEXT: v_cmp_eq_u32_e64 s9, 1, v20 +; GFX10-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:16 +; GFX10-NEXT: v_cmp_eq_u32_e64 s10, 1, v18 +; GFX10-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:12 +; GFX10-NEXT: v_cmp_eq_u32_e64 s11, 1, v16 +; GFX10-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:8 +; GFX10-NEXT: v_cmp_eq_u32_e64 s12, 1, v14 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:68 +; GFX10-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:24 +; GFX10-NEXT: v_cmp_eq_u32_e64 s13, 1, v12 +; GFX10-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:4 +; GFX10-NEXT: v_writelane_b32 v31, s30, 0 +; GFX10-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX10-NEXT: v_and_b32_e32 v2, 1, v2 +; GFX10-NEXT: v_and_b32_e32 v4, 1, v4 +; GFX10-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX10-NEXT: v_writelane_b32 v31, s31, 1 +; GFX10-NEXT: v_and_b32_e32 v8, 1, v8 +; GFX10-NEXT: v_and_b32_e32 v10, 1, v10 +; GFX10-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX10-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX10-NEXT: v_writelane_b32 v31, s34, 2 ; GFX10-NEXT: v_and_b32_e32 v5, 1, v5 -; GFX10-NEXT: v_cmp_eq_u32_e64 s15, 1, v20 -; GFX10-NEXT: v_cmp_eq_u32_e64 s16, 1, v22 -; GFX10-NEXT: v_cmp_eq_u32_e64 s17, 1, v24 -; GFX10-NEXT: v_cmp_eq_u32_e64 s18, 1, v26 -; GFX10-NEXT: v_cmp_eq_u32_e64 s19, 1, v28 -; GFX10-NEXT: v_cmp_eq_u32_e64 s20, 1, v30 -; GFX10-NEXT: v_cmp_eq_u32_e64 s22, 1, v7 -; GFX10-NEXT: v_cmp_eq_u32_e64 s23, 1, v9 -; GFX10-NEXT: v_cmp_eq_u32_e64 s27, 1, v17 +; GFX10-NEXT: v_and_b32_e32 v7, 1, v7 +; GFX10-NEXT: v_and_b32_e32 v9, 1, v9 +; GFX10-NEXT: v_and_b32_e32 v11, 1, v11 +; GFX10-NEXT: v_and_b32_e32 v13, 1, v13 +; GFX10-NEXT: v_and_b32_e32 v15, 1, v15 +; GFX10-NEXT: v_and_b32_e32 v17, 1, v17 +; GFX10-NEXT: v_and_b32_e32 v19, 1, v19 +; GFX10-NEXT: v_and_b32_e32 v21, 1, v21 +; GFX10-NEXT: v_and_b32_e32 v23, 1, v23 +; GFX10-NEXT: v_and_b32_e32 v25, 1, v25 +; GFX10-NEXT: v_and_b32_e32 v27, 1, v27 +; GFX10-NEXT: v_cmp_eq_u32_e64 s14, 1, v10 +; GFX10-NEXT: v_cmp_eq_u32_e64 s15, 1, v8 +; GFX10-NEXT: v_cmp_eq_u32_e64 s16, 1, v6 +; GFX10-NEXT: v_cmp_eq_u32_e64 s17, 1, v4 +; GFX10-NEXT: v_cmp_eq_u32_e64 s18, 1, v2 +; GFX10-NEXT: v_cmp_eq_u32_e64 s19, 1, v0 ; GFX10-NEXT: v_writelane_b32 v31, s35, 3 -; GFX10-NEXT: v_cmp_eq_u32_e64 s14, 1, v18 -; GFX10-NEXT: v_cmp_eq_u32_e64 s21, 1, v5 -; GFX10-NEXT: v_cmp_eq_u32_e64 s24, 1, v11 -; GFX10-NEXT: v_cmp_eq_u32_e64 s25, 1, v13 +; GFX10-NEXT: v_cmp_eq_u32_e64 s20, 1, v27 +; GFX10-NEXT: v_cmp_eq_u32_e64 s21, 1, v25 +; GFX10-NEXT: v_cmp_eq_u32_e64 s22, 1, v23 +; GFX10-NEXT: v_cmp_eq_u32_e64 s23, 1, v21 +; GFX10-NEXT: v_cmp_eq_u32_e64 s24, 1, v19 +; GFX10-NEXT: v_cmp_eq_u32_e64 s25, 1, v17 ; GFX10-NEXT: v_cmp_eq_u32_e64 s26, 1, v15 -; GFX10-NEXT: v_cmp_eq_u32_e64 s28, 1, v19 -; GFX10-NEXT: v_cmp_eq_u32_e64 s29, 1, v21 -; GFX10-NEXT: v_cmp_eq_u32_e64 s30, 1, v23 -; GFX10-NEXT: v_cmp_eq_u32_e64 s31, 1, v25 -; GFX10-NEXT: v_cmp_eq_u32_e64 s34, 1, v27 -; GFX10-NEXT: v_cmp_eq_u32_e64 s35, 1, v29 +; GFX10-NEXT: v_cmp_eq_u32_e64 s27, 1, v13 +; GFX10-NEXT: v_cmp_eq_u32_e64 s28, 1, v11 +; GFX10-NEXT: v_cmp_eq_u32_e64 s29, 1, v7 +; GFX10-NEXT: v_cmp_eq_u32_e64 s30, 1, v3 +; GFX10-NEXT: v_cmp_eq_u32_e64 s31, 1, v1 +; GFX10-NEXT: v_cmp_eq_u32_e64 s34, 1, v5 +; GFX10-NEXT: v_cmp_eq_u32_e64 s35, 1, v9 ; GFX10-NEXT: s_waitcnt vmcnt(32) -; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v32 +; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v32 ; GFX10-NEXT: s_waitcnt vmcnt(31) -; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v33 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v32, v33, s4 +; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v33 +; GFX10-NEXT: s_waitcnt vmcnt(30) +; GFX10-NEXT: v_and_b32_e32 v2, 1, v34 ; GFX10-NEXT: s_waitcnt vmcnt(29) -; GFX10-NEXT: v_cndmask_b32_e64 v11, v34, v35, s5 +; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v35 ; GFX10-NEXT: s_waitcnt vmcnt(28) -; GFX10-NEXT: v_and_b32_e32 v17, 1, v36 -; GFX10-NEXT: v_lshrrev_b32_e32 v13, 16, v35 -; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v34 -; GFX10-NEXT: s_waitcnt vmcnt(26) -; GFX10-NEXT: v_cndmask_b32_e64 v18, v37, v38, s7 -; GFX10-NEXT: v_lshrrev_b32_e32 v19, 16, v38 -; GFX10-NEXT: v_lshrrev_b32_e32 v20, 16, v37 +; GFX10-NEXT: v_cndmask_b32_e64 v15, v35, v36, s4 +; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v36 +; GFX10-NEXT: v_cndmask_b32_e64 v17, v33, v32, s5 +; GFX10-NEXT: s_waitcnt vmcnt(25) +; GFX10-NEXT: v_cndmask_b32_e64 v19, v38, v39, s7 ; GFX10-NEXT: s_waitcnt vmcnt(24) -; GFX10-NEXT: v_cndmask_b32_e64 v21, v39, v48, s8 -; GFX10-NEXT: v_lshrrev_b32_e32 v22, 16, v48 -; GFX10-NEXT: v_lshrrev_b32_e32 v23, 16, v39 -; GFX10-NEXT: s_waitcnt vmcnt(22) -; GFX10-NEXT: v_cndmask_b32_e64 v24, v50, v49, s9 -; GFX10-NEXT: v_lshrrev_b32_e32 v25, 16, v49 -; GFX10-NEXT: v_lshrrev_b32_e32 v26, 16, v50 -; GFX10-NEXT: s_waitcnt vmcnt(20) -; GFX10-NEXT: v_cndmask_b32_e64 v27, v51, v52, s10 -; GFX10-NEXT: v_lshrrev_b32_e32 v28, 16, v52 -; GFX10-NEXT: v_lshrrev_b32_e32 v29, 16, v51 +; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v48 +; GFX10-NEXT: s_waitcnt vmcnt(23) +; GFX10-NEXT: v_cndmask_b32_e64 v13, v48, v49, s6 +; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v49 +; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v39 +; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v38 +; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v37 ; GFX10-NEXT: s_waitcnt vmcnt(18) -; GFX10-NEXT: v_cndmask_b32_e64 v30, v53, v54, s11 +; GFX10-NEXT: v_cndmask_b32_e64 v27, v53, v54, s10 +; GFX10-NEXT: s_waitcnt vmcnt(17) +; GFX10-NEXT: v_lshrrev_b32_e32 v25, 16, v55 +; GFX10-NEXT: s_waitcnt vmcnt(16) +; GFX10-NEXT: v_cndmask_b32_e64 v21, v55, v64, s9 +; GFX10-NEXT: s_waitcnt vmcnt(15) +; GFX10-NEXT: v_cndmask_b32_e64 v11, v65, v37, s8 +; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v65 +; GFX10-NEXT: v_lshrrev_b32_e32 v23, 16, v64 ; GFX10-NEXT: v_lshrrev_b32_e32 v32, 16, v54 ; GFX10-NEXT: v_lshrrev_b32_e32 v33, 16, v53 -; GFX10-NEXT: s_waitcnt vmcnt(16) -; GFX10-NEXT: v_cndmask_b32_e64 v34, v55, v64, s12 -; GFX10-NEXT: v_lshrrev_b32_e32 v35, 16, v64 -; GFX10-NEXT: v_lshrrev_b32_e32 v36, 16, v55 -; GFX10-NEXT: s_waitcnt vmcnt(12) -; GFX10-NEXT: v_cndmask_b32_e64 v37, v68, v65, s13 -; GFX10-NEXT: v_lshrrev_b32_e32 v38, 16, v65 -; GFX10-NEXT: v_lshrrev_b32_e32 v39, 16, v68 -; GFX10-NEXT: v_lshrrev_b32_e32 v49, 16, v67 -; GFX10-NEXT: v_lshrrev_b32_e32 v50, 16, v66 +; GFX10-NEXT: v_cndmask_b32_e64 v34, v51, v52, s11 +; GFX10-NEXT: v_lshrrev_b32_e32 v35, 16, v52 +; GFX10-NEXT: v_lshrrev_b32_e32 v36, 16, v51 ; GFX10-NEXT: s_waitcnt vmcnt(9) -; GFX10-NEXT: v_cndmask_b32_e64 v52, v0, v2, s16 -; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v37, v30, v50, s12 +; GFX10-NEXT: v_lshrrev_b32_e32 v38, 16, v50 +; GFX10-NEXT: v_lshrrev_b32_e32 v30, 16, v30 +; GFX10-NEXT: v_cndmask_b32_e64 v39, v29, v69, s13 +; GFX10-NEXT: v_lshrrev_b32_e32 v48, 16, v69 +; GFX10-NEXT: v_lshrrev_b32_e32 v29, 16, v29 ; GFX10-NEXT: s_waitcnt vmcnt(6) -; GFX10-NEXT: v_cndmask_b32_e64 v53, v8, v69, s17 -; GFX10-NEXT: v_lshrrev_b32_e32 v54, 16, v69 -; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v55, v4, v3, s18 -; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v4 -; GFX10-NEXT: s_waitcnt vmcnt(3) -; GFX10-NEXT: v_cndmask_b32_e64 v64, v10, v12, s19 -; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v12 -; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_cndmask_b32_e64 v51, v1, v6, s15 -; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v6 -; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v65, v14, v16, s20 +; GFX10-NEXT: v_cndmask_b32_e64 v50, v24, v22, s15 +; GFX10-NEXT: v_lshrrev_b32_e32 v22, 16, v22 +; GFX10-NEXT: v_lshrrev_b32_e32 v24, 16, v24 +; GFX10-NEXT: s_waitcnt vmcnt(5) +; GFX10-NEXT: v_cndmask_b32_e64 v51, v68, v20, s16 +; GFX10-NEXT: v_lshrrev_b32_e32 v20, 16, v20 +; GFX10-NEXT: v_lshrrev_b32_e32 v52, 16, v68 +; GFX10-NEXT: s_waitcnt vmcnt(4) +; GFX10-NEXT: v_cndmask_b32_e64 v53, v67, v18, s17 +; GFX10-NEXT: v_lshrrev_b32_e32 v18, 16, v18 +; GFX10-NEXT: s_waitcnt vmcnt(1) +; GFX10-NEXT: v_cndmask_b32_e64 v49, v28, v26, s14 +; GFX10-NEXT: v_lshrrev_b32_e32 v26, 16, v26 +; GFX10-NEXT: v_lshrrev_b32_e32 v28, 16, v28 +; GFX10-NEXT: v_lshrrev_b32_e32 v54, 16, v67 +; GFX10-NEXT: v_cndmask_b32_e64 v55, v66, v16, s18 ; GFX10-NEXT: v_lshrrev_b32_e32 v16, 16, v16 +; GFX10-NEXT: v_lshrrev_b32_e32 v64, 16, v66 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_cndmask_b32_e64 v65, v14, v12, s19 +; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v12 ; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v14 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v17 -; GFX10-NEXT: v_cndmask_b32_e64 v48, v66, v67, s14 -; GFX10-NEXT: v_cndmask_b32_e64 v9, v15, v13, s6 -; GFX10-NEXT: v_cndmask_b32_e64 v13, v20, v19, s21 -; GFX10-NEXT: v_cndmask_b32_e64 v15, v23, v22, s22 -; GFX10-NEXT: v_cndmask_b32_e64 v19, v26, v25, s23 -; GFX10-NEXT: v_cndmask_b32_e64 v20, v29, v28, s24 -; GFX10-NEXT: v_cndmask_b32_e64 v22, v33, v32, s25 -; GFX10-NEXT: v_cndmask_b32_e64 v23, v36, v35, s26 -; GFX10-NEXT: v_cndmask_b32_e64 v25, v39, v38, s27 -; GFX10-NEXT: v_cndmask_b32_e64 v26, v50, v49, s28 -; GFX10-NEXT: v_cndmask_b32_e64 v28, v1, v6, s29 -; GFX10-NEXT: v_cndmask_b32_e64 v17, v0, v2, s30 -; GFX10-NEXT: v_cndmask_b32_e64 v29, v8, v54, s31 -; GFX10-NEXT: v_cndmask_b32_e64 v32, v4, v3, s34 -; GFX10-NEXT: v_cndmask_b32_e64 v33, v10, v12, s35 -; GFX10-NEXT: v_cndmask_b32_e32 v16, v14, v16, vcc_lo -; GFX10-NEXT: v_perm_b32 v0, v7, v5, 0x5040100 -; GFX10-NEXT: v_perm_b32 v1, v9, v11, 0x5040100 -; GFX10-NEXT: v_perm_b32 v2, v13, v18, 0x5040100 -; GFX10-NEXT: v_perm_b32 v3, v15, v21, 0x5040100 -; GFX10-NEXT: v_perm_b32 v4, v19, v24, 0x5040100 -; GFX10-NEXT: v_perm_b32 v5, v20, v27, 0x5040100 -; GFX10-NEXT: v_perm_b32 v6, v22, v30, 0x5040100 -; GFX10-NEXT: v_perm_b32 v7, v23, v34, 0x5040100 -; GFX10-NEXT: v_perm_b32 v8, v25, v37, 0x5040100 -; GFX10-NEXT: v_perm_b32 v9, v26, v48, 0x5040100 -; GFX10-NEXT: v_perm_b32 v10, v28, v51, 0x5040100 -; GFX10-NEXT: v_perm_b32 v11, v17, v52, 0x5040100 -; GFX10-NEXT: v_perm_b32 v12, v29, v53, 0x5040100 -; GFX10-NEXT: v_perm_b32 v13, v32, v55, 0x5040100 -; GFX10-NEXT: v_perm_b32 v14, v33, v64, 0x5040100 -; GFX10-NEXT: v_perm_b32 v15, v16, v65, 0x5040100 +; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 1, v2 +; GFX10-NEXT: v_cndmask_b32_e32 v66, v1, v0, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v67, v6, v5, s20 +; GFX10-NEXT: v_cndmask_b32_e64 v68, v8, v7, s21 +; GFX10-NEXT: v_cndmask_b32_e64 v69, v10, v9, s22 +; GFX10-NEXT: v_cndmask_b32_e64 v10, v25, v23, s23 +; GFX10-NEXT: v_cndmask_b32_e64 v9, v33, v32, s24 +; GFX10-NEXT: v_cndmask_b32_e64 v8, v36, v35, s25 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v30, v38, s26 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v29, v48, s27 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v28, v26, s28 +; GFX10-NEXT: v_cndmask_b32_e64 v20, v52, v20, s29 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v14, v12, s31 +; GFX10-NEXT: v_cndmask_b32_e64 v1, v64, v16, s30 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v54, v18, s34 +; GFX10-NEXT: v_cndmask_b32_e64 v12, v24, v22, s35 +; GFX10-NEXT: v_cndmask_b32_e64 v16, v4, v3, s4 +; GFX10-NEXT: v_perm_b32 v0, v0, v65, 0x5040100 +; GFX10-NEXT: v_perm_b32 v1, v1, v55, 0x5040100 +; GFX10-NEXT: v_perm_b32 v2, v2, v53, 0x5040100 +; GFX10-NEXT: v_perm_b32 v3, v20, v51, 0x5040100 +; GFX10-NEXT: v_perm_b32 v4, v12, v50, 0x5040100 +; GFX10-NEXT: v_perm_b32 v5, v5, v49, 0x5040100 +; GFX10-NEXT: v_perm_b32 v6, v6, v39, 0x5040100 +; GFX10-NEXT: v_perm_b32 v7, v7, v37, 0x5040100 +; GFX10-NEXT: v_perm_b32 v8, v8, v34, 0x5040100 +; GFX10-NEXT: v_perm_b32 v9, v9, v27, 0x5040100 +; GFX10-NEXT: v_perm_b32 v10, v10, v21, 0x5040100 +; GFX10-NEXT: v_perm_b32 v11, v69, v11, 0x5040100 +; GFX10-NEXT: v_perm_b32 v12, v68, v19, 0x5040100 +; GFX10-NEXT: v_perm_b32 v13, v67, v13, 0x5040100 +; GFX10-NEXT: v_perm_b32 v14, v66, v17, 0x5040100 +; GFX10-NEXT: v_perm_b32 v15, v16, v15, 0x5040100 ; GFX10-NEXT: v_readlane_b32 s35, v31, 3 ; GFX10-NEXT: v_readlane_b32 s34, v31, 2 ; GFX10-NEXT: v_readlane_b32 s31, v31, 1 @@ -30035,205 +29315,198 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x20 ; GFX11-NEXT: scratch_load_u16 v31, off, s32 -; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:68 -; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:4 -; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:72 -; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:8 -; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:76 -; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:12 -; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:80 -; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:16 -; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:84 -; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:20 -; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:88 -; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:24 -; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:92 -; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:28 -; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:96 -; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:32 -; GFX11-NEXT: scratch_load_b32 v64, off, s32 offset:100 -; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:36 -; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:104 -; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:40 -; GFX11-NEXT: scratch_load_b32 v68, off, s32 offset:108 -; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:44 -; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:112 -; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:48 -; GFX11-NEXT: scratch_load_b32 v80, off, s32 offset:116 -; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:52 -; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:120 -; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:56 -; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:124 -; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:60 -; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:128 -; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:64 -; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 -; GFX11-NEXT: v_and_b32_e32 v2, 1, v2 -; GFX11-NEXT: v_and_b32_e32 v4, 1, v4 -; GFX11-NEXT: v_and_b32_e32 v6, 1, v6 -; GFX11-NEXT: v_and_b32_e32 v8, 1, v8 -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 -; GFX11-NEXT: v_and_b32_e32 v27, 1, v27 -; GFX11-NEXT: v_and_b32_e32 v10, 1, v10 -; GFX11-NEXT: v_and_b32_e32 v12, 1, v12 -; GFX11-NEXT: v_and_b32_e32 v14, 1, v14 +; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:128 +; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:64 +; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:124 +; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:60 +; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:120 +; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:56 +; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:116 +; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:52 +; GFX11-NEXT: scratch_load_b32 v48, off, s32 offset:112 +; GFX11-NEXT: scratch_load_b32 v49, off, s32 offset:48 +; GFX11-NEXT: scratch_load_b32 v50, off, s32 offset:108 +; GFX11-NEXT: scratch_load_b32 v51, off, s32 offset:44 +; GFX11-NEXT: scratch_load_b32 v52, off, s32 offset:104 +; GFX11-NEXT: scratch_load_b32 v53, off, s32 offset:40 +; GFX11-NEXT: scratch_load_b32 v54, off, s32 offset:100 +; GFX11-NEXT: scratch_load_b32 v55, off, s32 offset:36 +; GFX11-NEXT: scratch_load_b32 v64, off, s32 offset:96 +; GFX11-NEXT: scratch_load_b32 v65, off, s32 offset:32 +; GFX11-NEXT: scratch_load_b32 v66, off, s32 offset:92 +; GFX11-NEXT: scratch_load_b32 v67, off, s32 offset:28 +; GFX11-NEXT: scratch_load_b32 v68, off, s32 offset:88 +; GFX11-NEXT: scratch_load_b32 v69, off, s32 offset:24 +; GFX11-NEXT: scratch_load_b32 v70, off, s32 offset:84 +; GFX11-NEXT: scratch_load_b32 v71, off, s32 offset:20 +; GFX11-NEXT: scratch_load_b32 v80, off, s32 offset:80 +; GFX11-NEXT: scratch_load_b32 v81, off, s32 offset:16 +; GFX11-NEXT: scratch_load_b32 v82, off, s32 offset:76 +; GFX11-NEXT: scratch_load_b32 v83, off, s32 offset:12 +; GFX11-NEXT: scratch_load_b32 v84, off, s32 offset:72 +; GFX11-NEXT: scratch_load_b32 v85, off, s32 offset:8 +; GFX11-NEXT: scratch_load_b32 v86, off, s32 offset:68 +; GFX11-NEXT: scratch_load_b32 v87, off, s32 offset:4 +; GFX11-NEXT: v_and_b32_e32 v30, 1, v30 +; GFX11-NEXT: v_and_b32_e32 v28, 1, v28 +; GFX11-NEXT: v_and_b32_e32 v26, 1, v26 +; GFX11-NEXT: v_and_b32_e32 v24, 1, v24 +; GFX11-NEXT: v_and_b32_e32 v22, 1, v22 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v30 +; GFX11-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX11-NEXT: v_and_b32_e32 v20, 1, v20 +; GFX11-NEXT: v_and_b32_e32 v18, 1, v18 +; GFX11-NEXT: v_and_b32_e32 v16, 1, v16 ; GFX11-NEXT: s_waitcnt vmcnt(30) -; GFX11-NEXT: v_cndmask_b32_e32 v0, v32, v33, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 -; GFX11-NEXT: v_and_b32_e32 v29, 1, v29 +; GFX11-NEXT: v_cndmask_b32_e32 v30, v32, v33, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v28 +; GFX11-NEXT: v_and_b32_e32 v1, 1, v1 ; GFX11-NEXT: v_lshrrev_b32_e32 v33, 16, v33 ; GFX11-NEXT: v_lshrrev_b32_e32 v32, 16, v32 -; GFX11-NEXT: v_and_b32_e32 v30, 1, v30 +; GFX11-NEXT: v_and_b32_e32 v0, 1, v0 ; GFX11-NEXT: s_waitcnt vmcnt(28) -; GFX11-NEXT: v_cndmask_b32_e32 v2, v34, v35, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 -; GFX11-NEXT: v_and_b32_e32 v23, 1, v23 +; GFX11-NEXT: v_cndmask_b32_e32 v28, v34, v35, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v26 +; GFX11-NEXT: v_and_b32_e32 v7, 1, v7 ; GFX11-NEXT: v_lshrrev_b32_e32 v35, 16, v35 ; GFX11-NEXT: v_lshrrev_b32_e32 v34, 16, v34 -; GFX11-NEXT: v_and_b32_e32 v28, 1, v28 +; GFX11-NEXT: v_and_b32_e32 v2, 1, v2 ; GFX11-NEXT: s_waitcnt vmcnt(26) -; GFX11-NEXT: v_cndmask_b32_e32 v4, v36, v37, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 -; GFX11-NEXT: v_and_b32_e32 v25, 1, v25 +; GFX11-NEXT: v_cndmask_b32_e32 v26, v36, v37, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v24 +; GFX11-NEXT: v_and_b32_e32 v5, 1, v5 ; GFX11-NEXT: v_lshrrev_b32_e32 v37, 16, v37 ; GFX11-NEXT: v_lshrrev_b32_e32 v36, 16, v36 -; GFX11-NEXT: v_and_b32_e32 v26, 1, v26 +; GFX11-NEXT: v_and_b32_e32 v4, 1, v4 ; GFX11-NEXT: s_waitcnt vmcnt(24) -; GFX11-NEXT: v_cndmask_b32_e32 v6, v38, v39, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v8 -; GFX11-NEXT: v_and_b32_e32 v19, 1, v19 +; GFX11-NEXT: v_cndmask_b32_e32 v24, v38, v39, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v22 +; GFX11-NEXT: v_and_b32_e32 v11, 1, v11 ; GFX11-NEXT: v_lshrrev_b32_e32 v39, 16, v39 ; GFX11-NEXT: v_lshrrev_b32_e32 v38, 16, v38 -; GFX11-NEXT: v_and_b32_e32 v24, 1, v24 +; GFX11-NEXT: v_and_b32_e32 v6, 1, v6 ; GFX11-NEXT: s_waitcnt vmcnt(22) -; GFX11-NEXT: v_cndmask_b32_e32 v8, v48, v49, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v10 -; GFX11-NEXT: v_and_b32_e32 v21, 1, v21 +; GFX11-NEXT: v_cndmask_b32_e32 v22, v48, v49, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v20 +; GFX11-NEXT: v_and_b32_e32 v9, 1, v9 ; GFX11-NEXT: v_lshrrev_b32_e32 v49, 16, v49 ; GFX11-NEXT: v_lshrrev_b32_e32 v48, 16, v48 -; GFX11-NEXT: v_and_b32_e32 v22, 1, v22 +; GFX11-NEXT: v_and_b32_e32 v8, 1, v8 ; GFX11-NEXT: s_waitcnt vmcnt(20) -; GFX11-NEXT: v_cndmask_b32_e32 v10, v50, v51, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v12 +; GFX11-NEXT: v_cndmask_b32_e32 v20, v50, v51, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v18 ; GFX11-NEXT: v_and_b32_e32 v15, 1, v15 ; GFX11-NEXT: v_lshrrev_b32_e32 v51, 16, v51 ; GFX11-NEXT: v_lshrrev_b32_e32 v50, 16, v50 -; GFX11-NEXT: v_and_b32_e32 v20, 1, v20 +; GFX11-NEXT: v_and_b32_e32 v10, 1, v10 ; GFX11-NEXT: s_waitcnt vmcnt(18) -; GFX11-NEXT: v_cndmask_b32_e32 v12, v52, v53, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14 -; GFX11-NEXT: v_and_b32_e32 v17, 1, v17 +; GFX11-NEXT: v_cndmask_b32_e32 v18, v52, v53, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v16 +; GFX11-NEXT: v_and_b32_e32 v13, 1, v13 ; GFX11-NEXT: v_lshrrev_b32_e32 v53, 16, v53 ; GFX11-NEXT: v_lshrrev_b32_e32 v52, 16, v52 -; GFX11-NEXT: v_and_b32_e32 v18, 1, v18 +; GFX11-NEXT: v_and_b32_e32 v12, 1, v12 ; GFX11-NEXT: s_waitcnt vmcnt(16) -; GFX11-NEXT: v_cndmask_b32_e32 v14, v54, v55, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v16, v54, v55, vcc_lo ; GFX11-NEXT: v_lshrrev_b32_e32 v55, 16, v55 ; GFX11-NEXT: v_lshrrev_b32_e32 v54, 16, v54 -; GFX11-NEXT: v_and_b32_e32 v16, 1, v16 +; GFX11-NEXT: v_and_b32_e32 v14, 1, v14 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v16 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14 ; GFX11-NEXT: s_waitcnt vmcnt(14) -; GFX11-NEXT: v_dual_cndmask_b32 v16, v64, v65 :: v_dual_and_b32 v11, 1, v11 -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v18 -; GFX11-NEXT: v_and_b32_e32 v13, 1, v13 +; GFX11-NEXT: v_dual_cndmask_b32 v14, v64, v65 :: v_dual_and_b32 v19, 1, v19 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v12 +; GFX11-NEXT: v_and_b32_e32 v17, 1, v17 ; GFX11-NEXT: v_lshrrev_b32_e32 v65, 16, v65 ; GFX11-NEXT: v_lshrrev_b32_e32 v64, 16, v64 ; GFX11-NEXT: s_waitcnt vmcnt(12) -; GFX11-NEXT: v_cndmask_b32_e32 v18, v66, v67, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v20 -; GFX11-NEXT: v_and_b32_e32 v7, 1, v7 +; GFX11-NEXT: v_cndmask_b32_e32 v12, v66, v67, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v10 +; GFX11-NEXT: v_and_b32_e32 v23, 1, v23 ; GFX11-NEXT: v_lshrrev_b32_e32 v67, 16, v67 ; GFX11-NEXT: v_lshrrev_b32_e32 v66, 16, v66 ; GFX11-NEXT: s_waitcnt vmcnt(10) -; GFX11-NEXT: v_cndmask_b32_e32 v20, v68, v69, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v22 -; GFX11-NEXT: v_and_b32_e32 v9, 1, v9 +; GFX11-NEXT: v_cndmask_b32_e32 v10, v68, v69, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v8 +; GFX11-NEXT: v_and_b32_e32 v21, 1, v21 ; GFX11-NEXT: v_lshrrev_b32_e32 v69, 16, v69 ; GFX11-NEXT: v_lshrrev_b32_e32 v68, 16, v68 ; GFX11-NEXT: s_waitcnt vmcnt(8) -; GFX11-NEXT: v_cndmask_b32_e32 v22, v70, v71, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v24 -; GFX11-NEXT: v_and_b32_e32 v3, 1, v3 +; GFX11-NEXT: v_cndmask_b32_e32 v8, v70, v71, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6 +; GFX11-NEXT: v_and_b32_e32 v27, 1, v27 ; GFX11-NEXT: v_lshrrev_b32_e32 v71, 16, v71 ; GFX11-NEXT: v_lshrrev_b32_e32 v70, 16, v70 ; GFX11-NEXT: s_waitcnt vmcnt(6) -; GFX11-NEXT: v_cndmask_b32_e32 v24, v80, v81, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v26 -; GFX11-NEXT: v_and_b32_e32 v5, 1, v5 +; GFX11-NEXT: v_cndmask_b32_e32 v6, v80, v81, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4 +; GFX11-NEXT: v_and_b32_e32 v25, 1, v25 ; GFX11-NEXT: v_lshrrev_b32_e32 v81, 16, v81 ; GFX11-NEXT: v_lshrrev_b32_e32 v80, 16, v80 ; GFX11-NEXT: s_waitcnt vmcnt(4) -; GFX11-NEXT: v_cndmask_b32_e32 v26, v82, v83, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v28 +; GFX11-NEXT: v_cndmask_b32_e32 v4, v82, v83, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2 ; GFX11-NEXT: v_and_b32_e32 v31, 1, v31 ; GFX11-NEXT: v_lshrrev_b32_e32 v83, 16, v83 ; GFX11-NEXT: v_lshrrev_b32_e32 v82, 16, v82 ; GFX11-NEXT: s_waitcnt vmcnt(2) -; GFX11-NEXT: v_cndmask_b32_e32 v28, v84, v85, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v30 -; GFX11-NEXT: v_and_b32_e32 v1, 1, v1 +; GFX11-NEXT: v_cndmask_b32_e32 v2, v84, v85, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX11-NEXT: v_and_b32_e32 v29, 1, v29 ; GFX11-NEXT: v_lshrrev_b32_e32 v85, 16, v85 ; GFX11-NEXT: v_lshrrev_b32_e32 v84, 16, v84 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_cndmask_b32_e32 v30, v86, v87, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 +; GFX11-NEXT: v_cndmask_b32_e32 v0, v86, v87, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v31 ; GFX11-NEXT: v_lshrrev_b32_e32 v87, 16, v87 ; GFX11-NEXT: v_lshrrev_b32_e32 v86, 16, v86 -; GFX11-NEXT: v_cndmask_b32_e32 v1, v32, v33, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v31, v32, v33, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v29 +; GFX11-NEXT: v_cndmask_b32_e32 v29, v34, v35, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v27 +; GFX11-NEXT: v_cndmask_b32_e32 v27, v36, v37, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v25 +; GFX11-NEXT: v_cndmask_b32_e32 v25, v38, v39, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v23 +; GFX11-NEXT: v_cndmask_b32_e32 v23, v48, v49, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v21 +; GFX11-NEXT: v_cndmask_b32_e32 v21, v50, v51, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v19 +; GFX11-NEXT: v_cndmask_b32_e32 v19, v52, v53, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v17 +; GFX11-NEXT: v_cndmask_b32_e32 v17, v54, v55, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v15 +; GFX11-NEXT: v_cndmask_b32_e32 v15, v64, v65, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13 +; GFX11-NEXT: v_cndmask_b32_e32 v13, v66, v67, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v11 +; GFX11-NEXT: v_cndmask_b32_e32 v11, v68, v69, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 +; GFX11-NEXT: v_cndmask_b32_e32 v7, v80, v81, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v3, v34, v35, vcc_lo +; GFX11-NEXT: v_cndmask_b32_e32 v3, v84, v85, vcc_lo +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1 +; GFX11-NEXT: v_cndmask_b32_e32 v1, v86, v87, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5 -; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v5, v36, v37, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v2, v5, v4, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v7, v38, v39, vcc_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 +; GFX11-NEXT: v_cndmask_b32_e32 v5, v82, v83, vcc_lo ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v9 +; GFX11-NEXT: v_perm_b32 v1, v3, v2, 0x5040100 ; GFX11-NEXT: v_perm_b32 v3, v7, v6, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v9, v48, v49, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v11 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v4, v9, v8, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v11, v50, v51, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13 -; GFX11-NEXT: v_perm_b32 v5, v11, v10, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v13, v52, v53, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v15 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_perm_b32 v6, v13, v12, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v15, v54, v55, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v17 +; GFX11-NEXT: v_perm_b32 v2, v5, v4, 0x5040100 +; GFX11-NEXT: v_cndmask_b32_e32 v9, v70, v71, vcc_lo +; GFX11-NEXT: v_perm_b32 v5, v11, v10, 0x5040100 ; GFX11-NEXT: v_perm_b32 v7, v15, v14, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v17, v64, v65, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v19 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v8, v17, v16, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v19, v66, v67, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v21 -; GFX11-NEXT: v_perm_b32 v9, v19, v18, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v21, v68, v69, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v23 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_perm_b32 v10, v21, v20, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v23, v70, v71, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v25 ; GFX11-NEXT: v_perm_b32 v11, v23, v22, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v25, v80, v81, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v27 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v4, v9, v8, 0x5040100 +; GFX11-NEXT: v_perm_b32 v8, v17, v16, 0x5040100 +; GFX11-NEXT: v_perm_b32 v9, v19, v18, 0x5040100 ; GFX11-NEXT: v_perm_b32 v12, v25, v24, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v27, v82, v83, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v29 ; GFX11-NEXT: v_perm_b32 v13, v27, v26, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v29, v84, v85, vcc_lo -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v31 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_perm_b32 v14, v29, v28, 0x5040100 -; GFX11-NEXT: v_cndmask_b32_e32 v31, v86, v87, vcc_lo ; GFX11-NEXT: v_perm_b32 v15, v31, v30, 0x5040100 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = select <32 x i1> %cond, <32 x bfloat> %a, <32 x bfloat> %b @@ -30446,98 +29719,61 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> ; GFX8-LABEL: v_fma_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_fma_f32 v1, v1, v3, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_fma_f32 v0, v0, v2, v4 -; GFX8-NEXT: v_fma_f32 v6, v8, v7, v6 +; GFX8-NEXT: v_fma_f32 v3, v6, v5, v3 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v6, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX8-NEXT: v_fma_f32 v2, v6, v4, v2 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v5 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_fma_f32 v1, v1, v3, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fma_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_fma_f32 v1, v1, v3, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_fma_f32 v6, v8, v7, v6 +; GFX9-NEXT: v_fma_f32 v3, v6, v5, v3 ; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v6, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX9-NEXT: v_fma_f32 v2, v6, v4, v2 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v5 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_fma_f32 v1, v1, v3, v4 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fma_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4 ; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v2 ; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_fmac_f32_e32 v6, v8, v7 ; GFX10-NEXT: v_fmac_f32_e32 v4, v0, v2 -; GFX10-NEXT: v_fmac_f32_e32 v9, v11, v10 ; GFX10-NEXT: v_fmac_f32_e32 v5, v1, v3 ; GFX10-NEXT: v_perm_b32 v0, v4, v6, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v5, v9, 0x7060302 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_fma_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_fmac_f32 v5, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_fmac_f32_e32 v4, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_fmac_f32 v9, v11, v10 :: v_dual_fmac_f32 v6, v8, v7 -; GFX11-NEXT: v_perm_b32 v1, v5, v9, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v4, v6, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <3 x bfloat> @llvm.fma.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) ret <3 x bfloat> %op } @@ -30596,97 +29832,97 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> ; GFX8-LABEL: v_fma_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_fma_f32 v0, v0, v2, v4 -; GFX8-NEXT: v_fma_f32 v6, v8, v7, v6 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v6, 16 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX8-NEXT: v_fma_f32 v2, v6, v4, v2 -; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_fma_f32 v1, v1, v3, v4 +; GFX8-NEXT: v_fma_f32 v6, v8, v7, v6 +; GFX8-NEXT: v_fma_f32 v1, v1, v3, v5 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_fma_f32 v0, v0, v2, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_fma_f32 v3, v7, v5, v3 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fma_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_fma_f32 v6, v8, v7, v6 +; GFX9-NEXT: v_fma_f32 v1, v1, v3, v5 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_fma_f32 v6, v8, v7, v6 +; GFX9-NEXT: v_fma_f32 v3, v7, v5, v3 ; GFX9-NEXT: v_fma_f32 v0, v0, v2, v4 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_perm_b32 v0, v0, v6, s4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v5 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v1 -; GFX9-NEXT: v_fma_f32 v2, v6, v4, v2 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v5 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_fma_f32 v1, v1, v3, v4 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fma_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v4 +; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v2 +; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_fmac_f32_e32 v6, v8, v7 -; GFX10-NEXT: v_fmac_f32_e32 v4, v0, v2 ; GFX10-NEXT: v_fmac_f32_e32 v9, v11, v10 +; GFX10-NEXT: v_fmac_f32_e32 v4, v0, v2 ; GFX10-NEXT: v_fmac_f32_e32 v5, v1, v3 -; GFX10-NEXT: v_perm_b32 v0, v4, v6, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v5, v9, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v4, v9, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v5, v6, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fma_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v5 -; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v9, 16, v4 +; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_fmac_f32 v9, v11, v10 :: v_dual_lshlrev_b32 v6, 16, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v3 +; GFX11-NEXT: v_dual_fmac_f32 v4, v0, v2 :: v_dual_and_b32 v3, 0xffff0000, v3 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v1 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_fmac_f32 v5, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_fmac_f32_e32 v4, v0, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_fmac_f32 v9, v11, v10 :: v_dual_fmac_f32 v6, v8, v7 -; GFX11-NEXT: v_perm_b32 v1, v5, v9, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v4, v6, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v4, v9, 0x7060302 +; GFX11-NEXT: v_fmac_f32_e32 v5, v1, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v6, v8, v7 +; GFX11-NEXT: v_perm_b32 v1, v5, v6, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <4 x bfloat> @llvm.fma.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) ret <4 x bfloat> %op @@ -30948,133 +30184,79 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl ; GFX8-LABEL: v_fmuladd_v3bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v5 +; GFX8-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2 +; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3 ; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v4 -; GFX8-NEXT: v_mul_f32_e32 v6, v7, v6 -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v4 -; GFX8-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_mul_f32_e32 v2, v4, v2 -; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v5 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX8-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_add_f32_e32 v6, v6, v7 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_add_f32_e32 v0, v0, v2 +; GFX8-NEXT: v_add_f32_e32 v3, v3, v5 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_add_f32_e32 v2, v2, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v6, 16 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fmuladd_v3bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v5 +; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_mul_f32_e32 v3, v5, v3 ; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v4 -; GFX9-NEXT: v_mul_f32_e32 v6, v7, v6 -; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v4 +; GFX9-NEXT: v_add_f32_e32 v3, v3, v5 ; GFX9-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_mul_f32_e32 v2, v4, v2 -; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v5 -; GFX9-NEXT: v_add_f32_e32 v6, v6, v7 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_add_f32_e32 v2, v2, v4 -; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v0, v0, v6, s4 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fmuladd_v3bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2 ; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_mul_f32_e32 v6, v7, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v6 -; GFX10-NEXT: v_mul_f32_e32 v6, v8, v7 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v4 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v4 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5 +; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX10-NEXT: v_mul_f32_e32 v3, v7, v6 +; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_add_f32_e32 v2, v2, v7 -; GFX10-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX10-NEXT: v_add_f32_e32 v3, v4, v6 +; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v5 +; GFX10-NEXT: v_add_f32_e32 v2, v2, v3 +; GFX10-NEXT: v_add_f32_e32 v0, v0, v4 +; GFX10-NEXT: v_alignbit_b32 v1, s4, v1, 16 ; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] -; -; GFX11-LABEL: v_fmuladd_v3bf16: -; GFX11: ; %bb.0: -; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX11-NEXT: v_dual_mul_f32 v6, v7, v6 :: v_dual_lshlrev_b32 v7, 16, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_mul_f32_e32 v6, v8, v7 -; GFX11-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v4 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v5 -; GFX11-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_and_b32 v1, 0xffff0000, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_dual_add_f32 v2, v2, v7 :: v_dual_and_b32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_add_f32_e32 v3, v4, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 -; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <3 x bfloat> @llvm.fmuladd.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c) ret <3 x bfloat> %op } @@ -31149,132 +30331,131 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl ; GFX8-LABEL: v_fmuladd_v4bf16: ; GFX8: ; %bb.0: ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v4 -; GFX8-NEXT: v_mul_f32_e32 v6, v7, v6 -; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v4 -; GFX8-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v1 +; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v1 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX8-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX8-NEXT: v_mul_f32_e32 v2, v4, v2 ; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v5 -; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v5 +; GFX8-NEXT: v_mul_f32_e32 v6, v7, v6 +; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v5 ; GFX8-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2 +; GFX8-NEXT: v_mul_f32_e32 v3, v5, v3 +; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v4 +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX8-NEXT: v_add_f32_e32 v0, v0, v2 ; GFX8-NEXT: v_add_f32_e32 v6, v6, v7 -; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX8-NEXT: v_add_f32_e32 v2, v2, v4 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX8-NEXT: v_alignbit_b32 v0, v0, v6, 16 -; GFX8-NEXT: v_alignbit_b32 v1, v1, v2, 16 +; GFX8-NEXT: v_add_f32_e32 v3, v3, v5 +; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0 +; GFX8-NEXT: v_alignbit_b32 v0, v0, v3, 16 +; GFX8-NEXT: v_alignbit_b32 v1, v1, v6, 16 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: v_fmuladd_v4bf16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v4 -; GFX9-NEXT: v_mul_f32_e32 v6, v7, v6 -; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v4 -; GFX9-NEXT: v_add_f32_e32 v0, v0, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v1 +; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v1 ; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_mul_f32_e32 v2, v4, v2 ; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v5 +; GFX9-NEXT: v_mul_f32_e32 v6, v7, v6 +; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v5 +; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v0 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_mul_f32_e32 v3, v5, v3 +; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2 +; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v4 +; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v4 ; GFX9-NEXT: v_add_f32_e32 v6, v6, v7 +; GFX9-NEXT: v_add_f32_e32 v3, v3, v5 +; GFX9-NEXT: v_add_f32_e32 v0, v0, v2 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: v_add_f32_e32 v2, v2, v4 -; GFX9-NEXT: v_add_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_perm_b32 v0, v0, v6, s4 -; GFX9-NEXT: v_perm_b32 v1, v1, v2, s4 +; GFX9-NEXT: v_perm_b32 v0, v0, v3, s4 +; GFX9-NEXT: v_perm_b32 v1, v1, v6, s4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: v_fmuladd_v4bf16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v0 -; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v0 +; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 ; GFX10-NEXT: v_mul_f32_e32 v6, v7, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v6 -; GFX10-NEXT: v_mul_f32_e32 v6, v8, v7 +; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v3 +; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX10-NEXT: v_mul_f32_e32 v3, v8, v7 +; GFX10-NEXT: v_mul_f32_e32 v0, v0, v2 +; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v5 +; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v4 +; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v4 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v6 -; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5 -; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX10-NEXT: v_add_f32_e32 v2, v2, v7 -; GFX10-NEXT: v_add_f32_e32 v0, v0, v3 -; GFX10-NEXT: v_add_f32_e32 v3, v4, v6 +; GFX10-NEXT: v_add_f32_e32 v2, v6, v2 +; GFX10-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX10-NEXT: v_add_f32_e32 v0, v0, v4 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX10-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v0, v3, 0x7060302 +; GFX10-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: v_fmuladd_v4bf16: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v1 -; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v0 +; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0 ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_dual_mul_f32 v6, v7, v6 :: v_dual_and_b32 v3, 0xffff0000, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v2 +; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; GFX11-NEXT: v_mul_f32_e32 v0, v0, v2 -; GFX11-NEXT: v_dual_mul_f32 v6, v7, v6 :: v_dual_lshlrev_b32 v7, 16, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v5 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_mul_f32_e32 v6, v8, v7 -; GFX11-NEXT: v_mul_f32_e32 v1, v1, v3 -; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-NEXT: v_dual_add_f32 v2, v6, v2 :: v_dual_mul_f32 v1, v1, v3 +; GFX11-NEXT: v_mul_f32_e32 v3, v8, v7 ; GFX11-NEXT: v_lshlrev_b32_e32 v7, 16, v4 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v6 -; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v5 -; GFX11-NEXT: v_dual_add_f32 v0, v0, v3 :: v_dual_and_b32 v1, 0xffff0000, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX11-NEXT: v_dual_add_f32 v2, v2, v7 :: v_dual_and_b32 v5, 0xffff0000, v5 -; GFX11-NEXT: v_add_f32_e32 v3, v4, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_and_b32 v3, 0xffff0000, v3 ; GFX11-NEXT: v_add_f32_e32 v1, v1, v5 -; GFX11-NEXT: v_perm_b32 v0, v0, v2, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_add_f32_e32 v3, v3, v7 +; GFX11-NEXT: v_perm_b32 v1, v1, v2, 0x7060302 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v1, v1, v3, 0x7060302 +; GFX11-NEXT: v_perm_b32 v0, v0, v3, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %op = call <4 x bfloat> @llvm.fmuladd.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) ret <4 x bfloat> %op diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll index ef2be37d62d9..160bae51193e 100644 --- a/llvm/test/CodeGen/AMDGPU/function-args.ll +++ b/llvm/test/CodeGen/AMDGPU/function-args.ll @@ -3165,11 +3165,11 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[4:7], 0 ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:8 -; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:12 -; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:20 -; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:16 -; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4 +; VI-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:4 +; VI-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:8 +; VI-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:20 +; VI-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:16 +; VI-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:12 ; VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0 @@ -3178,16 +3178,14 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: buffer_store_dword v20, off, s[4:7], 0 -; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: buffer_store_dword v16, off, s[4:7], 0 -; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: buffer_store_dword v17, off, s[4:7], 0 -; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: buffer_store_dword v18, off, s[4:7], 0 ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: buffer_store_dword v19, off, s[4:7], 0 ; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: buffer_store_dword v20, off, s[4:7], 0 +; VI-NEXT: s_waitcnt vmcnt(0) +; VI-NEXT: buffer_store_dwordx2 v[16:17], off, s[4:7], 0 +; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: s_setpc_b64 s[30:31] ; ; GFX9-LABEL: void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16: @@ -3205,11 +3203,11 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: buffer_store_dwordx4 v[16:19], off, s[4:7], 0 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:8 -; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:12 -; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:20 -; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:16 -; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:4 +; GFX9-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:4 +; GFX9-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:8 +; GFX9-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:20 +; GFX9-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:16 +; GFX9-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:12 ; GFX9-NEXT: s_nop 0 ; GFX9-NEXT: buffer_store_dwordx4 v[12:15], off, s[4:7], 0 ; GFX9-NEXT: s_waitcnt vmcnt(0) @@ -3219,16 +3217,14 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: buffer_store_dword v20, off, s[4:7], 0 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: buffer_store_dword v16, off, s[4:7], 0 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: buffer_store_dword v17, off, s[4:7], 0 -; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: buffer_store_dword v18, off, s[4:7], 0 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: buffer_store_dword v19, off, s[4:7], 0 ; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: buffer_store_dword v20, off, s[4:7], 0 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: buffer_store_dwordx2 v[16:17], off, s[4:7], 0 +; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16: @@ -3236,11 +3232,11 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_clause 0x5 ; GFX11-NEXT: scratch_load_b32 v31, off, s32 -; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4 -; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8 -; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:12 -; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:20 -; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:16 +; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:20 +; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:4 +; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:8 +; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:12 +; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:16 ; GFX11-NEXT: s_mov_b32 s3, 0x31016000 ; GFX11-NEXT: s_mov_b32 s2, -1 ; GFX11-NEXT: s_waitcnt vmcnt(5) @@ -3260,20 +3256,17 @@ define void @void_func_v32i32_v2i16_v2f16_v2bf16_v4bf16(<32 x i32> %arg0, <2 x i ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 -; GFX11-NEXT: s_waitcnt vmcnt(4) -; GFX11-NEXT: buffer_store_b32 v32, off, s[0:3], 0 dlc -; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: s_waitcnt vmcnt(3) -; GFX11-NEXT: buffer_store_b32 v33, off, s[0:3], 0 dlc +; GFX11-NEXT: buffer_store_b32 v34, off, s[0:3], 0 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: s_waitcnt vmcnt(2) -; GFX11-NEXT: buffer_store_b32 v34, off, s[0:3], 0 dlc +; GFX11-NEXT: buffer_store_b32 v35, off, s[0:3], 0 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: buffer_store_b32 v35, off, s[0:3], 0 dlc +; GFX11-NEXT: buffer_store_b32 v36, off, s[0:3], 0 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: buffer_store_b32 v36, off, s[0:3], 0 dlc +; GFX11-NEXT: buffer_store_b64 v[32:33], off, s[0:3], 0 dlc ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: s_setpc_b64 s[30:31] store volatile <32 x i32> %arg0, ptr addrspace(1) undef diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll index ca3336beccf7..8dcd0f504e50 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll @@ -948,57 +948,52 @@ define <3 x i1> @isnan_v3bf16(<3 x bfloat> %x) nounwind { ; GFX8CHECK-LABEL: isnan_v3bf16: ; GFX8CHECK: ; %bb.0: ; GFX8CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8CHECK-NEXT: v_mov_b32_e32 v2, 0x7fff -; GFX8CHECK-NEXT: v_and_b32_sdwa v3, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD +; GFX8CHECK-NEXT: v_and_b32_e32 v2, 0x7fff, v1 ; GFX8CHECK-NEXT: s_movk_i32 s4, 0x7f80 -; GFX8CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0 -; GFX8CHECK-NEXT: v_and_b32_e32 v1, 0x7fff, v1 +; GFX8CHECK-NEXT: v_bfe_u32 v1, v0, 16, 15 +; GFX8CHECK-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0 ; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v0 ; GFX8CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc ; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v1 -; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v3 ; GFX8CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc +; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v2 +; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; GFX8CHECK-NEXT: s_setpc_b64 s[30:31] ; ; GFX9CHECK-LABEL: isnan_v3bf16: ; GFX9CHECK: ; %bb.0: ; GFX9CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9CHECK-NEXT: s_movk_i32 s4, 0x7fff -; GFX9CHECK-NEXT: v_and_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD +; GFX9CHECK-NEXT: v_and_b32_e32 v3, 0x7fff7fff, v0 ; GFX9CHECK-NEXT: s_movk_i32 s4, 0x7f80 -; GFX9CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0 ; GFX9CHECK-NEXT: v_and_b32_e32 v1, 0x7fff, v1 -; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v0 +; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v3 ; GFX9CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc ; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v1 +; GFX9CHECK-NEXT: v_cmp_gt_i16_sdwa s[4:5], v3, s4 src0_sel:WORD_1 src1_sel:DWORD ; GFX9CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v3 -; GFX9CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc +; GFX9CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] ; GFX9CHECK-NEXT: s_setpc_b64 s[30:31] ; ; GFX10CHECK-LABEL: isnan_v3bf16: ; GFX10CHECK: ; %bb.0: ; GFX10CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10CHECK-NEXT: v_mov_b32_e32 v2, 0x7fff -; GFX10CHECK-NEXT: v_and_b32_e32 v3, 0x7fff, v0 +; GFX10CHECK-NEXT: v_and_b32_e32 v2, 0x7fff7fff, v0 +; GFX10CHECK-NEXT: v_mov_b32_e32 v3, 0x7f80 ; GFX10CHECK-NEXT: v_and_b32_e32 v4, 0x7fff, v1 -; GFX10CHECK-NEXT: v_and_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3 -; GFX10CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2 -; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX10CHECK-NEXT: v_cmp_gt_i16_sdwa s4, v2, v3 src0_sel:WORD_1 src1_sel:DWORD +; GFX10CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v4 +; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4 ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo ; GFX10CHECK-NEXT: s_setpc_b64 s[30:31] ; ; GFX11CHECK-LABEL: isnan_v3bf16: ; GFX11CHECK: ; %bb.0: ; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11CHECK-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX11CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0 +; GFX11CHECK-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0 ; GFX11CHECK-NEXT: v_and_b32_e32 v3, 0x7fff, v1 -; GFX11CHECK-NEXT: v_and_b32_e32 v2, 0x7fff, v2 +; GFX11CHECK-NEXT: v_lshrrev_b32_e32 v2, 16, v0 ; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0 ; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2 @@ -1032,18 +1027,17 @@ define <4 x i1> @isnan_v4bf16(<4 x bfloat> %x) nounwind { ; GFX8CHECK-LABEL: isnan_v4bf16: ; GFX8CHECK: ; %bb.0: ; GFX8CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX8CHECK-NEXT: v_mov_b32_e32 v2, 0x7fff +; GFX8CHECK-NEXT: v_bfe_u32 v3, v1, 16, 15 ; GFX8CHECK-NEXT: s_movk_i32 s4, 0x7f80 -; GFX8CHECK-NEXT: v_and_b32_sdwa v4, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX8CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0 -; GFX8CHECK-NEXT: v_and_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX8CHECK-NEXT: v_and_b32_e32 v1, 0x7fff, v1 +; GFX8CHECK-NEXT: v_and_b32_e32 v2, 0x7fff7fff, v1 +; GFX8CHECK-NEXT: v_bfe_u32 v1, v0, 16, 15 +; GFX8CHECK-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0 ; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v0 ; GFX8CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc ; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v1 -; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v4 ; GFX8CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc +; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v2 +; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; GFX8CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s4, v3 ; GFX8CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX8CHECK-NEXT: s_setpc_b64 s[30:31] @@ -1051,56 +1045,49 @@ define <4 x i1> @isnan_v4bf16(<4 x bfloat> %x) nounwind { ; GFX9CHECK-LABEL: isnan_v4bf16: ; GFX9CHECK: ; %bb.0: ; GFX9CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9CHECK-NEXT: s_movk_i32 s4, 0x7fff -; GFX9CHECK-NEXT: s_movk_i32 s5, 0x7f80 -; GFX9CHECK-NEXT: v_and_b32_sdwa v4, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX9CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0 -; GFX9CHECK-NEXT: v_and_b32_sdwa v3, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX9CHECK-NEXT: v_and_b32_e32 v1, 0x7fff, v1 -; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s5, v0 +; GFX9CHECK-NEXT: v_and_b32_e32 v3, 0x7fff7fff, v1 +; GFX9CHECK-NEXT: s_movk_i32 s6, 0x7f80 +; GFX9CHECK-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0 +; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s6, v1 +; GFX9CHECK-NEXT: v_cmp_gt_i16_sdwa s[4:5], v1, s6 src0_sel:WORD_1 src1_sel:DWORD ; GFX9CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc -; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s5, v1 +; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s6, v3 +; GFX9CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] +; GFX9CHECK-NEXT: v_cmp_gt_i16_sdwa s[4:5], v3, s6 src0_sel:WORD_1 src1_sel:DWORD ; GFX9CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s5, v4 -; GFX9CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc -; GFX9CHECK-NEXT: v_cmp_lt_i16_e32 vcc, s5, v3 -; GFX9CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc +; GFX9CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5] ; GFX9CHECK-NEXT: s_setpc_b64 s[30:31] ; ; GFX10CHECK-LABEL: isnan_v4bf16: ; GFX10CHECK: ; %bb.0: ; GFX10CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10CHECK-NEXT: v_and_b32_e32 v3, 0x7fff, v0 -; GFX10CHECK-NEXT: v_mov_b32_e32 v2, 0x7fff -; GFX10CHECK-NEXT: v_and_b32_e32 v4, 0x7fff, v1 +; GFX10CHECK-NEXT: v_and_b32_e32 v3, 0x7fff7fff, v0 +; GFX10CHECK-NEXT: v_mov_b32_e32 v5, 0x7f80 +; GFX10CHECK-NEXT: v_and_b32_e32 v4, 0x7fff7fff, v1 ; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3 -; GFX10CHECK-NEXT: v_and_b32_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD -; GFX10CHECK-NEXT: v_and_b32_sdwa v1, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD +; GFX10CHECK-NEXT: v_cmp_gt_i16_sdwa s4, v3, v5 src0_sel:WORD_1 src1_sel:DWORD ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v4 +; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4 +; GFX10CHECK-NEXT: v_cmp_gt_i16_sdwa s4, v4, v5 src0_sel:WORD_1 src1_sel:DWORD ; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo -; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1 -; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo -; GFX10CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v5 -; GFX10CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo +; GFX10CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4 ; GFX10CHECK-NEXT: s_setpc_b64 s[30:31] ; ; GFX11CHECK-LABEL: isnan_v4bf16: ; GFX11CHECK: ; %bb.0: ; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11CHECK-NEXT: v_lshrrev_b32_e32 v3, 16, v0 -; GFX11CHECK-NEXT: v_and_b32_e32 v0, 0x7fff, v0 -; GFX11CHECK-NEXT: v_lshrrev_b32_e32 v2, 16, v1 -; GFX11CHECK-NEXT: v_and_b32_e32 v1, 0x7fff, v1 -; GFX11CHECK-NEXT: v_and_b32_e32 v3, 0x7fff, v3 +; GFX11CHECK-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0 +; GFX11CHECK-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1 ; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0 -; GFX11CHECK-NEXT: v_and_b32_e32 v4, 0x7fff, v2 +; GFX11CHECK-NEXT: v_lshrrev_b32_e32 v4, 16, v0 +; GFX11CHECK-NEXT: v_lshrrev_b32_e32 v3, 16, v1 ; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo ; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1 ; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo -; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3 -; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo ; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v4 +; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo +; GFX11CHECK-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3 ; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo ; GFX11CHECK-NEXT: s_setpc_b64 s[30:31] %1 = call <4 x i1> @llvm.is.fpclass.v4bf16(<4 x bfloat> %x, i32 3) ; nan diff --git a/llvm/test/CodeGen/AMDGPU/select-undef.ll b/llvm/test/CodeGen/AMDGPU/select-undef.ll index 8ca33dce73f8..5b9866a3c915 100644 --- a/llvm/test/CodeGen/AMDGPU/select-undef.ll +++ b/llvm/test/CodeGen/AMDGPU/select-undef.ll @@ -303,9 +303,8 @@ ret: ret void } -; FIXME: This shouldn't have the 0 initialization ; GCN-LABEL: {{^}}undef_v3bf16: -; GCN: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} +; GCN-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}} ; GCN-NOT: s_mov_b32 s{{[0-9]+}}, 0{{$}} ; GCN: s_cbranch_vccnz define amdgpu_kernel void @undef_v3bf16(ptr addrspace(3) %ptr, i1 %cond) { diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll index dfe98bbbaddf..d76bb48b4a82 100644 --- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll +++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll @@ -2703,30 +2703,30 @@ define <4 x bfloat> @shuffle_v4bf16_234u(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_234u: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v5, v[2:3], off +; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_mov_b32_e32 v0, v6 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_mov_b32_e32 v1, v4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_234u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX10-NEXT: global_load_dword v5, v[2:3], off +; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_mov_b32_e32 v0, v6 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_234u: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 -; GFX11-NEXT: global_load_b32 v1, v[2:3], off +; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -2935,32 +2935,35 @@ define <4 x bfloat> @shuffle_v4bf16_357u(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_357u: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 +; GFX9-NEXT: s_waitcnt vmcnt(1) +; GFX9-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_perm_b32 v0, v4, v6, s4 -; GFX9-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_357u: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX10-NEXT: s_waitcnt vmcnt(1) +; GFX10-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v4, v6, 0x7060302 -; GFX10-NEXT: v_alignbit_b32 v1, s4, v5, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_357u: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:4 -; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off +; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off +; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 +; GFX11-NEXT: s_waitcnt vmcnt(1) +; GFX11-NEXT: v_alignbit_b32 v1, s0, v3, 16 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_perm_b32 v0, v0, v4, 0x7060302 -; GFX11-NEXT: v_alignbit_b32 v1, s0, v1, 16 +; GFX11-NEXT: v_perm_b32 v0, v2, v0, 0x7060302 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 @@ -2972,24 +2975,29 @@ define <4 x bfloat> @shuffle_v4bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v0, v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3029,31 +3037,39 @@ define <4 x bfloat> @shuffle_v4bf16_0145(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0145: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off +; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0145: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v4, v[0:1], off -; GFX10-NEXT: global_load_dword v5, v[2:3], off +; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0145: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off -; GFX11-NEXT: global_load_b32 v1, v[2:3], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off +; GFX11-NEXT: s_waitcnt vmcnt(1) +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 @@ -3065,30 +3081,33 @@ define <4 x bfloat> @shuffle_v4bf16_0167(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0167: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off +; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v5 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_mov_b32_e32 v1, v4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0167: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v4, v[0:1], off -; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off +; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v5 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0167: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4 +; GFX11-NEXT: s_waitcnt vmcnt(1) +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3102,7 +3121,9 @@ define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v1 ; GFX9-NEXT: v_mov_b32_e32 v0, v2 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -3111,6 +3132,7 @@ define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; @@ -3119,6 +3141,7 @@ define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3161,30 +3184,33 @@ define <4 x bfloat> @shuffle_v4bf16_2345(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_2345: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v5, v[2:3], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2345: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_2345: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 -; GFX11-NEXT: global_load_b32 v1, v[2:3], off +; GFX11-NEXT: s_waitcnt vmcnt(1) +; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v2, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3233,33 +3259,39 @@ define <4 x bfloat> @shuffle_v4bf16_4501(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_4501: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[2:3], off -; GFX9-NEXT: global_load_dword v5, v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4501: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v4, v[2:3], off -; GFX10-NEXT: global_load_dword v5, v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off +; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4501: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v2, v[2:3], off -; GFX11-NEXT: global_load_b32 v1, v[0:1], off +; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off +; GFX11-NEXT: global_load_b64 v[3:4], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v2, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v3, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 @@ -3271,32 +3303,33 @@ define <4 x bfloat> @shuffle_v4bf16_4523(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_4523: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[2:3], off -; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v5 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_mov_b32_e32 v1, v4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4523: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v4, v[2:3], off -; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v5 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_mov_b32_e32 v1, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4523: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v2, v[2:3], off +; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v2, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3309,24 +3342,29 @@ define <4 x bfloat> @shuffle_v4bf16_4545(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_4545: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[2:3], off +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4545: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v0, v[2:3], off +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4545: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[2:3], off +; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3366,32 +3404,33 @@ define <4 x bfloat> @shuffle_v4bf16_6701(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_6701: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX9-NEXT: global_load_dword v5, v[0:1], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6701: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_6701: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 -; GFX11-NEXT: global_load_b32 v1, v[0:1], off +; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off +; GFX11-NEXT: global_load_b32 v0, v[2:3], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v4, v4 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3443,7 +3482,9 @@ define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v1 ; GFX9-NEXT: v_mov_b32_e32 v0, v2 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; @@ -3452,6 +3493,7 @@ define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; @@ -3460,6 +3502,7 @@ define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3578,31 +3621,34 @@ define <4 x bfloat> @shuffle_v4bf16_3456(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_3456: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX9-NEXT: s_waitcnt vmcnt(1) +; GFX9-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v4, v6, 16 -; GFX9-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_3456: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 ; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX10-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX10-NEXT: s_waitcnt vmcnt(1) +; GFX10-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v4, v6, 16 -; GFX10-NEXT: v_alignbit_b32 v1, v5, v4, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_3456: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 -; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off +; GFX11-NEXT: s_waitcnt vmcnt(1) +; GFX11-NEXT: v_alignbit_b32 v1, v3, v2, 16 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_alignbit_b32 v0, v1, v0, 16 -; GFX11-NEXT: v_alignbit_b32 v1, v2, v1, 16 +; GFX11-NEXT: v_alignbit_b32 v0, v2, v0, 16 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 @@ -3693,7 +3739,7 @@ define <4 x bfloat> @shuffle_v4bf16_0000(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0000: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_mov_b32 s4, 0x5040100 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_perm_b32 v0, v0, v0, s4 @@ -3703,7 +3749,7 @@ define <4 x bfloat> @shuffle_v4bf16_0000(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX10-LABEL: shuffle_v4bf16_0000: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v0, v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v0, v0, 0x5040100 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 @@ -3712,7 +3758,7 @@ define <4 x bfloat> @shuffle_v4bf16_0000(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-LABEL: shuffle_v4bf16_0000: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_perm_b32 v0, v0, v0, 0x5040100 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -3728,7 +3774,7 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_1010: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 @@ -3737,7 +3783,7 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX10-LABEL: shuffle_v4bf16_1010: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v0, v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 @@ -3746,7 +3792,7 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-LABEL: shuffle_v4bf16_1010: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_alignbit_b32 v0, v0, v0, 16 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -3762,7 +3808,7 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_1100: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v1, v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX9-NEXT: s_mov_b32 s4, 0x7060302 ; GFX9-NEXT: s_mov_b32 s5, 0x5040100 ; GFX9-NEXT: s_waitcnt vmcnt(0) @@ -3773,7 +3819,7 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX10-LABEL: shuffle_v4bf16_1100: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v1, v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_perm_b32 v0, v1, v1, 0x7060302 ; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 @@ -3782,7 +3828,7 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-LABEL: shuffle_v4bf16_1100: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v1, v[0:1], off +; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_perm_b32 v0, v1, v1, 0x7060302 ; GFX11-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 @@ -3897,24 +3943,29 @@ define <4 x bfloat> @shuffle_v8bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v8bf16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff ; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v0, v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 @@ -3954,31 +4005,39 @@ define <4 x bfloat> @shuffle_v8bf16_4589(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v8bf16_4589: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:8 -; GFX9-NEXT: global_load_dword v5, v[2:3], off +; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:8 +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v5 +; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_4589: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:8 -; GFX10-NEXT: global_load_dword v5, v[2:3], off +; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:8 +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v5 +; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_4589: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:8 -; GFX11-NEXT: global_load_b32 v1, v[2:3], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:8 +; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off +; GFX11-NEXT: s_waitcnt vmcnt(1) +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 @@ -4067,9 +4126,11 @@ define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 +; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: s_mov_b32 s5, 0x5040100 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v1, v1, v1, s4 +; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 +; GFX9-NEXT: v_perm_b32 v1, v1, v1, s5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v3bf16_0122: @@ -4077,6 +4138,7 @@ define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; @@ -4085,6 +4147,7 @@ define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX11-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <3 x bfloat>, ptr addrspace(1) %arg0 @@ -4190,27 +4253,27 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl ; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX9-NEXT: v_lshlrev_b32_e32 v11, 16, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v1 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v1 +; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v3 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX9-NEXT: v_fma_f32 v7, v8, v9, v7 ; GFX9-NEXT: v_fma_f32 v0, v8, v4, v0 -; GFX9-NEXT: v_fma_f32 v4, v11, v4, v12 -; GFX9-NEXT: v_fma_f32 v1, v11, v9, v1 +; GFX9-NEXT: v_fma_f32 v8, v12, v9, v11 +; GFX9-NEXT: v_fma_f32 v1, v12, v4, v1 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 ; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v7 ; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v8 ; GFX9-NEXT: v_fma_f32 v0, v2, v10, v0 -; GFX9-NEXT: v_fma_f32 v2, v2, v5, v7 -; GFX9-NEXT: v_fma_f32 v1, v3, v5, v1 -; GFX9-NEXT: v_fma_f32 v3, v3, v10, v4 +; GFX9-NEXT: v_fma_f32 v2, v2, v5, v4 +; GFX9-NEXT: v_fma_f32 v1, v3, v10, v1 +; GFX9-NEXT: v_fma_f32 v3, v3, v5, v7 +; GFX9-NEXT: v_perm_b32 v1, v3, v1, s0 ; GFX9-NEXT: v_perm_b32 v0, v2, v0, s0 -; GFX9-NEXT: v_perm_b32 v1, v1, v3, s0 ; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7] ; GFX9-NEXT: s_endpgm ; @@ -4233,27 +4296,27 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl ; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v3 -; GFX10-NEXT: v_and_b32_e32 v12, 0xffff0000, v1 -; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v3 +; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v1 +; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX10-NEXT: v_fmac_f32_e32 v7, v8, v9 ; GFX10-NEXT: v_fmac_f32_e32 v0, v8, v4 ; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX10-NEXT: v_fmac_f32_e32 v12, v11, v9 -; GFX10-NEXT: v_fmac_f32_e32 v1, v11, v4 -; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v5 +; GFX10-NEXT: v_fmac_f32_e32 v11, v10, v4 +; GFX10-NEXT: v_fmac_f32_e32 v1, v10, v9 +; GFX10-NEXT: v_lshlrev_b32_e32 v12, 16, v5 ; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 ; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 ; GFX10-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v7 -; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v12 +; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v11 ; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX10-NEXT: v_fmac_f32_e32 v0, v2, v10 -; GFX10-NEXT: v_fmac_f32_e32 v4, v2, v5 -; GFX10-NEXT: v_fmac_f32_e32 v7, v3, v5 -; GFX10-NEXT: v_fmac_f32_e32 v1, v3, v10 -; GFX10-NEXT: v_perm_b32 v0, v4, v0, 0x7060302 -; GFX10-NEXT: v_perm_b32 v1, v7, v1, 0x7060302 +; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX10-NEXT: v_fmac_f32_e32 v0, v2, v12 +; GFX10-NEXT: v_fmac_f32_e32 v4, v3, v12 +; GFX10-NEXT: v_fmac_f32_e32 v1, v3, v5 +; GFX10-NEXT: v_fmac_f32_e32 v7, v2, v5 +; GFX10-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 +; GFX10-NEXT: v_perm_b32 v0, v7, v0, 0x7060302 ; GFX10-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7] ; GFX10-NEXT: s_endpgm ; @@ -4269,36 +4332,36 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl ; GFX11-NEXT: global_load_b64 v[2:3], v6, s[0:1] ; GFX11-NEXT: global_load_b64 v[4:5], v6, s[2:3] ; GFX11-NEXT: s_waitcnt vmcnt(2) -; GFX11-NEXT: v_and_b32_e32 v12, 0xffff0000, v1 -; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v3 +; GFX11-NEXT: v_lshlrev_b32_e32 v11, 16, v1 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5 ; GFX11-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 -; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX11-NEXT: v_lshlrev_b32_e32 v12, 16, v5 +; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v3 ; GFX11-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v2 -; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 -; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_fmac_f32 v1, v11, v4 :: v_dual_and_b32 v2, 0xffff0000, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v1, v10, v9 ; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_fmac_f32 v1, v3, v10 :: v_dual_fmac_f32 v0, v8, v4 +; GFX11-NEXT: v_dual_fmac_f32 v1, v3, v5 :: v_dual_lshlrev_b32 v4, 16, v4 +; GFX11-NEXT: v_dual_fmac_f32 v11, v10, v4 :: v_dual_lshlrev_b32 v8, 16, v2 +; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_and_b32 v2, 0xffff0000, v2 +; GFX11-NEXT: v_fmac_f32_e32 v0, v8, v4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v11 +; GFX11-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fmac_f32_e32 v0, v2, v10 -; GFX11-NEXT: v_fmac_f32_e32 v12, v11, v9 -; GFX11-NEXT: v_fmac_f32_e32 v7, v8, v9 -; GFX11-NEXT: v_and_b32_e32 v4, 0xffff0000, v7 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_dual_fmac_f32 v4, v2, v5 :: v_dual_and_b32 v7, 0xffff0000, v12 -; GFX11-NEXT: v_fmac_f32_e32 v7, v3, v5 +; GFX11-NEXT: v_dual_fmac_f32 v4, v3, v12 :: v_dual_fmac_f32 v7, v2, v5 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_perm_b32 v0, v4, v0, 0x7060302 -; GFX11-NEXT: v_perm_b32 v1, v7, v1, 0x7060302 +; GFX11-NEXT: v_fmac_f32_e32 v0, v2, v12 +; GFX11-NEXT: v_perm_b32 v1, v1, v4, 0x7060302 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-NEXT: v_perm_b32 v0, v7, v0, 0x7060302 ; GFX11-NEXT: global_store_b64 v6, v[0:1], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -4336,28 +4399,32 @@ define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0456: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v6, v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX9-NEXT: s_mov_b32 s4, 0x5040100 +; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v4, v6, s4 -; GFX9-NEXT: v_alignbit_b32 v1, v5, v4, 16 +; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 +; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0456: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dword v6, v[0:1], off -; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_perm_b32 v0, v4, v6, 0x5040100 -; GFX10-NEXT: v_alignbit_b32 v1, v5, v4, 16 +; GFX10-NEXT: v_perm_b32 v0, v5, v4, 0x5040100 +; GFX10-NEXT: v_alignbit_b32 v1, v6, v5, 16 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0456: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b32 v0, v[0:1], off +; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100 -- GitLab From 597086c60959dd5b3c032552e8b42dd1d053f233 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Fri, 5 Jan 2024 08:44:19 +0700 Subject: [PATCH 060/728] DAG: Implement promotion for strict_fp_round (#74332) Needs an AMDGPU hack to get the selection to work. The ordinary variant is custom lowered through an almost equivalent target node that would need a strict variant for additional known bits optimizations. --- .../include/llvm/Target/TargetSelectionDAG.td | 13 ++++ .../SelectionDAG/LegalizeFloatTypes.cpp | 46 ++++++++++++- .../SelectionDAG/LegalizeIntegerTypes.cpp | 15 ++++- llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 2 + llvm/lib/Target/AMDGPU/SIInstructions.td | 9 ++- llvm/test/CodeGen/AMDGPU/strict_fp_casts.ll | 67 +++++++++++++++++++ 6 files changed, 149 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index d4fc9d8a96db..22360353790d 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -614,6 +614,12 @@ def strict_sint_to_fp : SDNode<"ISD::STRICT_SINT_TO_FP", SDTIntToFPOp, [SDNPHasChain]>; def strict_uint_to_fp : SDNode<"ISD::STRICT_UINT_TO_FP", SDTIntToFPOp, [SDNPHasChain]>; + +def strict_f16_to_fp : SDNode<"ISD::STRICT_FP16_TO_FP", + SDTIntToFPOp, [SDNPHasChain]>; +def strict_fp_to_f16 : SDNode<"ISD::STRICT_FP_TO_FP16", + SDTFPToIntOp, [SDNPHasChain]>; + def strict_fsetcc : SDNode<"ISD::STRICT_FSETCC", SDTSetCC, [SDNPHasChain]>; def strict_fsetccs : SDNode<"ISD::STRICT_FSETCCS", SDTSetCC, [SDNPHasChain]>; @@ -1576,6 +1582,13 @@ def any_fsetccs : PatFrags<(ops node:$lhs, node:$rhs, node:$pred), [(strict_fsetccs node:$lhs, node:$rhs, node:$pred), (setcc node:$lhs, node:$rhs, node:$pred)]>; +def any_f16_to_fp : PatFrags<(ops node:$src), + [(f16_to_fp node:$src), + (strict_f16_to_fp node:$src)]>; +def any_fp_to_f16 : PatFrags<(ops node:$src), + [(fp_to_f16 node:$src), + (strict_fp_to_f16 node:$src)]>; + multiclass binary_atomic_op_ord { def NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val), (!cast(NAME) node:$ptr, node:$val)> { diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 65919a64b806..6e0e1e23419b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -2181,6 +2181,24 @@ static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { report_fatal_error("Attempt at an invalid promotion-related conversion"); } +static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) { + if (OpVT == MVT::f16) + return ISD::STRICT_FP16_TO_FP; + + if (RetVT == MVT::f16) + return ISD::STRICT_FP_TO_FP16; + + if (OpVT == MVT::bf16) { + // TODO: return ISD::STRICT_BF16_TO_FP; + } + + if (RetVT == MVT::bf16) { + // TODO: return ISD::STRICT_FP_TO_BF16; + } + + report_fatal_error("Attempt at an invalid promotion-related conversion"); +} + bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG)); SDValue R = SDValue(); @@ -2281,7 +2299,7 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, unsigned OpNo) { - assert(OpNo == 1); + assert(OpNo == 1 && "Promoting unpromotable operand"); SDValue Op = GetPromotedFloat(N->getOperand(1)); EVT VT = N->getValueType(0); @@ -2416,6 +2434,9 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; + case ISD::STRICT_FP_ROUND: + R = PromoteFloatRes_STRICT_FP_ROUND(N); + break; case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; @@ -2621,6 +2642,29 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); } +// Explicit operation to reduce precision. Reduce the value to half precision +// and promote it back to the legal type. +SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) { + SDLoc DL(N); + + SDValue Chain = N->getOperand(0); + SDValue Op = N->getOperand(1); + EVT VT = N->getValueType(0); + EVT OpVT = Op->getValueType(0); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); + + // Round promoted float to desired precision + SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL, + DAG.getVTList(IVT, MVT::Other), Chain, Op); + // Promote it back to the legal output type + SDValue Res = + DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL, + DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round); + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return Res; +} + SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { LoadSDNode *L = cast(N); EVT VT = N->getValueType(0); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 3d21bd22e6ef..92598d885619 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -165,7 +165,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16_BF16(N); break; - + case ISD::STRICT_FP_TO_FP16: + Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N); + break; case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; case ISD::AND: @@ -787,6 +789,16 @@ SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); } +SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) { + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + SDLoc dl(N); + + SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), + N->getOperand(0), N->getOperand(1)); + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return Res; +} + SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDLoc dl(N); @@ -1804,6 +1816,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::FP16_TO_FP: case ISD::VP_UINT_TO_FP: case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; + case ISD::STRICT_FP16_TO_FP: case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 84b1b2c71fd0..09f0bca8b861 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -326,6 +326,7 @@ private: SDValue PromoteIntRes_FP_TO_XINT(SDNode *N); SDValue PromoteIntRes_FP_TO_XINT_SAT(SDNode *N); SDValue PromoteIntRes_FP_TO_FP16_BF16(SDNode *N); + SDValue PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N); SDValue PromoteIntRes_XRINT(SDNode *N); SDValue PromoteIntRes_FREEZE(SDNode *N); SDValue PromoteIntRes_INT_EXTEND(SDNode *N); @@ -699,6 +700,7 @@ private: SDValue PromoteFloatRes_ExpOp(SDNode *N); SDValue PromoteFloatRes_FFREXP(SDNode *N); SDValue PromoteFloatRes_FP_ROUND(SDNode *N); + SDValue PromoteFloatRes_STRICT_FP_ROUND(SDNode *N); SDValue PromoteFloatRes_LOAD(SDNode *N); SDValue PromoteFloatRes_SELECT(SDNode *N); SDValue PromoteFloatRes_SELECT_CC(SDNode *N); diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index e8c4d805dbba..1158d16baa1b 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -1097,7 +1097,7 @@ def : Pat < multiclass f16_fp_Pats { // f16_to_fp patterns def : GCNPat < - (f32 (f16_to_fp i32:$src0)), + (f32 (any_f16_to_fp i32:$src0)), (cvt_f32_f16_inst_e64 SRCMODS.NONE, $src0) >; @@ -1151,6 +1151,13 @@ multiclass f16_fp_Pats; + + // This is only used on targets without half support + // TODO: Introduce strict variant of AMDGPUfp_to_f16 and share custom lowering + def : GCNPat < + (i32 (strict_fp_to_f16 (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))), + (cvt_f16_f32_inst_e64 $src0_modifiers, f32:$src0) + >; } let SubtargetPredicate = NotHasTrue16BitInsts in diff --git a/llvm/test/CodeGen/AMDGPU/strict_fp_casts.ll b/llvm/test/CodeGen/AMDGPU/strict_fp_casts.ll index 0339fca4d56c..8a3647a9b6e9 100644 --- a/llvm/test/CodeGen/AMDGPU/strict_fp_casts.ll +++ b/llvm/test/CodeGen/AMDGPU/strict_fp_casts.ll @@ -3,6 +3,9 @@ declare float @llvm.experimental.constrained.fpext.f32.f16(half, metadata) #0 declare <2 x float> @llvm.experimental.constrained.fpext.v2f32.v2f16(<2 x half>, metadata) #0 +declare half @llvm.experimental.constrained.fptrunc.f16.f32(float, metadata, metadata) #0 +declare <2 x half> @llvm.experimental.constrained.fptrunc.v2f16.v2f32(<2 x float>, metadata, metadata) #0 +declare float @llvm.fabs.f32(float) define float @v_constrained_fpext_f16_to_f32(ptr addrspace(1) %ptr) #0 { ; GFX7-LABEL: v_constrained_fpext_f16_to_f32: @@ -40,4 +43,68 @@ define <2 x float> @v_constrained_fpext_v2f16_to_v2f32(ptr addrspace(1) %ptr) #0 ret <2 x float> %result } +define void @v_constrained_fptrunc_f32_to_f16_fpexcept_strict(float %arg, ptr addrspace(1) %ptr) #0 { +; GFX7-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict: +; GFX7: ; %bb.0: +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 +; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX7-NEXT: s_setpc_b64 s[30:31] + %result = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %arg, metadata !"round.tonearest", metadata !"fpexcept.strict") + ret void +} + +define void @v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict(<2 x float> %arg, ptr addrspace(1) %ptr) #0 { +; GFX7-LABEL: v_constrained_fptrunc_v2f32_to_v2f16_fpexcept_strict: +; GFX7: ; %bb.0: +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 +; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 +; GFX7-NEXT: s_mov_b32 s6, 0 +; GFX7-NEXT: s_mov_b32 s7, 0xf000 +; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1 +; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX7-NEXT: s_mov_b32 s4, s6 +; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1 +; GFX7-NEXT: s_mov_b32 s5, s6 +; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0 +; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX7-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64 +; GFX7-NEXT: s_waitcnt vmcnt(0) +; GFX7-NEXT: s_setpc_b64 s[30:31] + %result = call <2 x half> @llvm.experimental.constrained.fptrunc.v2f16.v2f32(<2 x float> %arg, metadata !"round.tonearest", metadata !"fpexcept.strict") + store <2 x half> %result, ptr addrspace(1) %ptr + ret void +} + +define void @v_constrained_fptrunc_f32_to_f16_fpexcept_strict_fneg(float %arg, ptr addrspace(1) %ptr) #0 { +; GFX7-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_fneg: +; GFX7: ; %bb.0: +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e64 v0, -v0 +; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX7-NEXT: s_setpc_b64 s[30:31] + %neg.arg = fneg float %arg + %result = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %neg.arg, metadata !"round.tonearest", metadata !"fpexcept.strict") + ret void +} + +define void @v_constrained_fptrunc_f32_to_f16_fpexcept_strict_fabs(float %arg, ptr addrspace(1) %ptr) #0 { +; GFX7-LABEL: v_constrained_fptrunc_f32_to_f16_fpexcept_strict_fabs: +; GFX7: ; %bb.0: +; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX7-NEXT: v_cvt_f16_f32_e64 v0, |v0| +; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0 +; GFX7-NEXT: s_setpc_b64 s[30:31] + %abs.arg = call float @llvm.fabs.f32(float %arg) + %result = call half @llvm.experimental.constrained.fptrunc.f16.f32(float %abs.arg, metadata !"round.tonearest", metadata !"fpexcept.strict") + ret void +} + attributes #0 = { strictfp } -- GitLab From 2cf420d5b846a4733ef0ef7c8ed0ae0bfd1c6772 Mon Sep 17 00:00:00 2001 From: wanglei Date: Tue, 2 Jan 2024 10:55:02 +0800 Subject: [PATCH 061/728] [LoongArch] Emit function call code sequence as `PCADDU18I+JIRL` in medium code model According to the description of the psABI v2.20: https://github.com/loongson/la-abi-specs/releases/tag/v2.20, adjustments are made to the function call instructions under the medium code model. At the same time, AsmParser has already supported parsing the call36 and tail36 macro instructions. --- .../AsmParser/LoongArchAsmParser.cpp | 61 +++++++++++++++++++ .../LoongArch/LoongArchExpandPseudoInsts.cpp | 29 ++++----- .../Target/LoongArch/LoongArchInstrInfo.td | 23 ++++++- .../Target/LoongArch/LoongArchMCInstLower.cpp | 3 + .../LoongArch/LoongArchTargetMachine.cpp | 4 +- .../MCTargetDesc/LoongArchBaseInfo.h | 1 + .../MCTargetDesc/LoongArchELFObjectWriter.cpp | 2 + .../MCTargetDesc/LoongArchFixupKinds.h | 5 +- .../MCTargetDesc/LoongArchMCCodeEmitter.cpp | 3 + .../MCTargetDesc/LoongArchMCExpr.cpp | 3 + .../LoongArch/MCTargetDesc/LoongArchMCExpr.h | 1 + llvm/test/CodeGen/LoongArch/code-models.ll | 12 ++-- .../MC/LoongArch/Basic/Integer/invalid64.s | 2 +- llvm/test/MC/LoongArch/Macros/macros-call.s | 9 +++ .../MC/LoongArch/Relocations/relocations.s | 5 ++ 15 files changed, 134 insertions(+), 29 deletions(-) create mode 100644 llvm/test/MC/LoongArch/Macros/macros-call.s diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp index 66a37fce5dda..46f63a4103f9 100644 --- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp +++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp @@ -121,6 +121,10 @@ class LoongArchAsmParser : public MCTargetAsmParser { // Helper to emit pseudo instruction "li.w/d $rd, $imm". void emitLoadImm(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out); + // Helper to emit pseudo instruction "call36 sym" or "tail36 $rj, sym". + void emitFuncCall36(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out, + bool IsTailCall); + public: enum LoongArchMatchResultTy { Match_Dummy = FIRST_TARGET_MATCH_RESULT_TY, @@ -400,6 +404,22 @@ public: IsValidKind; } + bool isSImm20pcaddu18i() const { + if (!isImm()) + return false; + + int64_t Imm; + LoongArchMCExpr::VariantKind VK = LoongArchMCExpr::VK_LoongArch_None; + bool IsConstantImm = evaluateConstantImm(getImm(), Imm, VK); + bool IsValidKind = VK == LoongArchMCExpr::VK_LoongArch_None || + VK == LoongArchMCExpr::VK_LoongArch_CALL36; + + return IsConstantImm + ? isInt<20>(Imm) && IsValidKind + : LoongArchAsmParser::classifySymbolRef(getImm(), VK) && + IsValidKind; + } + bool isSImm21lsl2() const { if (!isImm()) return false; @@ -1110,6 +1130,35 @@ void LoongArchAsmParser::emitLoadImm(MCInst &Inst, SMLoc IDLoc, } } +void LoongArchAsmParser::emitFuncCall36(MCInst &Inst, SMLoc IDLoc, + MCStreamer &Out, bool IsTailCall) { + // call36 sym + // expands to: + // pcaddu18i $ra, %call36(sym) + // jirl $ra, $ra, 0 + // + // tail36 $rj, sym + // expands to: + // pcaddu18i $rj, %call36(sym) + // jirl $r0, $rj, 0 + unsigned ScratchReg = + IsTailCall ? Inst.getOperand(0).getReg() : (unsigned)LoongArch::R1; + const MCExpr *Sym = + IsTailCall ? Inst.getOperand(1).getExpr() : Inst.getOperand(0).getExpr(); + const LoongArchMCExpr *LE = LoongArchMCExpr::create( + Sym, llvm::LoongArchMCExpr::VK_LoongArch_CALL36, getContext()); + + Out.emitInstruction( + MCInstBuilder(LoongArch::PCADDU18I).addReg(ScratchReg).addExpr(LE), + getSTI()); + Out.emitInstruction( + MCInstBuilder(LoongArch::JIRL) + .addReg(IsTailCall ? (unsigned)LoongArch::R0 : ScratchReg) + .addReg(ScratchReg) + .addImm(0), + getSTI()); +} + bool LoongArchAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc, OperandVector &Operands, MCStreamer &Out) { @@ -1158,6 +1207,12 @@ bool LoongArchAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc, case LoongArch::PseudoLI_D: emitLoadImm(Inst, IDLoc, Out); return false; + case LoongArch::PseudoCALL36: + emitFuncCall36(Inst, IDLoc, Out, /*IsTailCall=*/false); + return false; + case LoongArch::PseudoTAIL36: + emitFuncCall36(Inst, IDLoc, Out, /*IsTailCall=*/true); + return false; } Out.emitInstruction(Inst, getSTI()); return false; @@ -1439,6 +1494,12 @@ bool LoongArchAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, /*Upper=*/(1 << 19) - 1, "operand must be a symbol with modifier (e.g. %pc_hi20) or an integer " "in the range"); + case Match_InvalidSImm20pcaddu18i: + return generateImmOutOfRangeError( + Operands, ErrorInfo, /*Lower=*/-(1 << 19), + /*Upper=*/(1 << 19) - 1, + "operand must be a symbol with modifier (e.g. %call36) or an integer " + "in the range"); case Match_InvalidSImm21lsl2: return generateImmOutOfRangeError( Operands, ErrorInfo, /*Lower=*/-(1 << 22), /*Upper=*/(1 << 22) - 4, diff --git a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp index 72c1f1cec198..8eda2dcc1633 100644 --- a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp @@ -458,11 +458,11 @@ bool LoongArchPreRAExpandPseudo::expandFunctionCALL( } case CodeModel::Medium: { // CALL: - // pcalau12i $ra, %pc_hi20(func) - // jirl $ra, $ra, %pc_lo12(func) + // pcaddu18i $ra, %call36(func) + // jirl $ra, $ra, 0 // TAIL: - // pcalau12i $scratch, %pc_hi20(func) - // jirl $r0, $scratch, %pc_lo12(func) + // pcaddu18i $scratch, %call36(func) + // jirl $r0, $scratch, 0 Opcode = IsTailCall ? LoongArch::PseudoJIRL_TAIL : LoongArch::PseudoJIRL_CALL; Register ScratchReg = @@ -470,18 +470,15 @@ bool LoongArchPreRAExpandPseudo::expandFunctionCALL( ? MF->getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass) : LoongArch::R1; MachineInstrBuilder MIB = - BuildMI(MBB, MBBI, DL, TII->get(LoongArch::PCALAU12I), ScratchReg); - CALL = BuildMI(MBB, MBBI, DL, TII->get(Opcode)).addReg(ScratchReg); - if (Func.isSymbol()) { - const char *FnName = Func.getSymbolName(); - MIB.addExternalSymbol(FnName, LoongArchII::MO_PCREL_HI); - CALL.addExternalSymbol(FnName, LoongArchII::MO_PCREL_LO); - break; - } - assert(Func.isGlobal() && "Expected a GlobalValue at this time"); - const GlobalValue *GV = Func.getGlobal(); - MIB.addGlobalAddress(GV, 0, LoongArchII::MO_PCREL_HI); - CALL.addGlobalAddress(GV, 0, LoongArchII::MO_PCREL_LO); + BuildMI(MBB, MBBI, DL, TII->get(LoongArch::PCADDU18I), ScratchReg); + + CALL = + BuildMI(MBB, MBBI, DL, TII->get(Opcode)).addReg(ScratchReg).addImm(0); + + if (Func.isSymbol()) + MIB.addExternalSymbol(Func.getSymbolName(), LoongArchII::MO_CALL36); + else + MIB.addDisp(Func, 0, LoongArchII::MO_CALL36); break; } case CodeModel::Large: { diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td index 2fea0f33e9eb..21555f885630 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td @@ -377,6 +377,10 @@ def simm20_lu32id : SImm20Operand { let ParserMatchClass = SImmAsmOperand<20, "lu32id">; } +def simm20_pcaddu18i : SImm20Operand { + let ParserMatchClass = SImmAsmOperand<20, "pcaddu18i">; +} + def simm21_lsl2 : Operand { let ParserMatchClass = SImmAsmOperand<21, "lsl2">; let EncoderMethod = "getImmOpValueAsr<2>"; @@ -832,7 +836,7 @@ def LU32I_D : Fmt1RI20<0x16000000, (outs GPR:$dst), "$rd, $imm20">; } def LU52I_D : ALU_2RI12<0x03000000, simm12_lu52id>; -def PCADDU18I : ALU_1RI20<0x1e000000, simm20>; +def PCADDU18I : ALU_1RI20<0x1e000000, simm20_pcaddu18i>; def MUL_D : ALU_3R<0x001d8000>; def MULH_D : ALU_3R<0x001e0000>; def MULH_DU : ALU_3R<0x001e8000>; @@ -1396,7 +1400,7 @@ def : Pat<(brind (add GPR:$rj, simm16_lsl2:$imm16)), (PseudoBRIND GPR:$rj, simm16_lsl2:$imm16)>; let isCall = 1, Defs = [R1] in -def PseudoCALL : Pseudo<(outs), (ins simm26_symbol:$func)>; +def PseudoCALL : Pseudo<(outs), (ins bare_symbol:$func)>; def : Pat<(loongarch_call tglobaladdr:$func), (PseudoCALL tglobaladdr:$func)>; def : Pat<(loongarch_call texternalsym:$func), (PseudoCALL texternalsym:$func)>; @@ -1416,7 +1420,7 @@ def PseudoRET : Pseudo<(outs), (ins), [(loongarch_ret)]>, PseudoInstExpansion<(JIRL R0, R1, 0)>; let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [R3] in -def PseudoTAIL : Pseudo<(outs), (ins simm26_symbol:$dst)>; +def PseudoTAIL : Pseudo<(outs), (ins bare_symbol:$dst)>; def : Pat<(loongarch_tail (iPTR tglobaladdr:$dst)), (PseudoTAIL tglobaladdr:$dst)>; @@ -1439,6 +1443,19 @@ def PseudoJIRL_TAIL : Pseudo<(outs), (ins GPR:$rj, simm16_lsl2:$imm16)>, PseudoInstExpansion<(JIRL R0, GPR:$rj, simm16_lsl2:$imm16)>; +/// call36/taill36 macro instructions +let isCall = 1, isBarrier = 1, isCodeGenOnly = 0, isAsmParserOnly = 1, + Defs = [R1], Size = 8, hasSideEffects = 0, mayStore = 0, mayLoad = 0 in +def PseudoCALL36 : Pseudo<(outs), (ins bare_symbol:$dst), [], + "call36", "$dst">, + Requires<[IsLA64]>; +let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [R3], + isCodeGenOnly = 0, isAsmParserOnly = 1, Size = 8, hasSideEffects = 0, + mayStore = 0, mayLoad = 0 in +def PseudoTAIL36 : Pseudo<(outs), (ins GPR:$tmp, bare_symbol:$dst), [], + "tail36", "$tmp, $dst">, + Requires<[IsLA64]>; + /// Load address (la*) macro instructions. // Define isCodeGenOnly = 0 to expose them to tablegened assembly parser. diff --git a/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp b/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp index 5daa9481c907..98ad49f25e3f 100644 --- a/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp @@ -95,6 +95,9 @@ static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym, case LoongArchII::MO_GD_PC_HI: Kind = LoongArchMCExpr::VK_LoongArch_TLS_GD_PC_HI20; break; + case LoongArchII::MO_CALL36: + Kind = LoongArchMCExpr::VK_LoongArch_CALL36; + break; // TODO: Handle more target-flags. } diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp index a5a4d78aceee..62ae1dea00d6 100644 --- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp @@ -63,11 +63,11 @@ getEffectiveLoongArchCodeModel(const Triple &TT, switch (*CM) { case CodeModel::Small: - case CodeModel::Medium: return *CM; + case CodeModel::Medium: case CodeModel::Large: if (!TT.isArch64Bit()) - report_fatal_error("Large code model requires LA64"); + report_fatal_error("Medium/Large code model requires LA64"); return *CM; default: report_fatal_error( diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h index cee6dad1f095..0692cb92b694 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchBaseInfo.h @@ -47,6 +47,7 @@ enum { MO_IE_PC64_HI, MO_LD_PC_HI, MO_GD_PC_HI, + MO_CALL36 // TODO: Add more flags. }; } // end namespace LoongArchII diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp index fe19a4f2d3c8..1dec816f3473 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp @@ -90,6 +90,8 @@ unsigned LoongArchELFObjectWriter::getRelocType(MCContext &Ctx, return ELF::R_LARCH_TLS_LE64_LO20; case LoongArch::fixup_loongarch_tls_le64_hi12: return ELF::R_LARCH_TLS_LE64_HI12; + case LoongArch::fixup_loongarch_call36: + return ELF::R_LARCH_CALL36; // TODO: Handle more fixup-kinds. } } diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h index 178fa6e5262b..e827bae1f3e3 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchFixupKinds.h @@ -108,7 +108,10 @@ enum Fixups { // 20-bit fixup corresponding to %gd_hi20(foo) for instruction lu12i.w. fixup_loongarch_tls_gd_hi20, // Generate an R_LARCH_RELAX which indicates the linker may relax here. - fixup_loongarch_relax = FirstLiteralRelocationKind + ELF::R_LARCH_RELAX + fixup_loongarch_relax = FirstLiteralRelocationKind + ELF::R_LARCH_RELAX, + // 36-bit fixup corresponding to %call36(foo) for a pair instructions: + // pcaddu18i+jirl. + fixup_loongarch_call36 = FirstLiteralRelocationKind + ELF::R_LARCH_CALL36, }; } // end namespace LoongArch } // end namespace llvm diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp index d2ea062dc09a..9ac0128f2517 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp @@ -241,6 +241,9 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO, case LoongArchMCExpr::VK_LoongArch_TLS_GD_HI20: FixupKind = LoongArch::fixup_loongarch_tls_gd_hi20; break; + case LoongArchMCExpr::VK_LoongArch_CALL36: + FixupKind = LoongArch::fixup_loongarch_call36; + break; } } else if (Kind == MCExpr::SymbolRef && cast(Expr)->getKind() == diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp index 82c992b1cc8c..8ca8876a19b9 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp @@ -138,6 +138,8 @@ StringRef LoongArchMCExpr::getVariantKindName(VariantKind Kind) { return "gd_pc_hi20"; case VK_LoongArch_TLS_GD_HI20: return "gd_hi20"; + case VK_LoongArch_CALL36: + return "call36"; } } @@ -180,6 +182,7 @@ LoongArchMCExpr::getVariantKindForName(StringRef name) { .Case("ld_hi20", VK_LoongArch_TLS_LD_HI20) .Case("gd_pc_hi20", VK_LoongArch_TLS_GD_PC_HI20) .Case("gd_hi20", VK_LoongArch_TLS_GD_HI20) + .Case("call36", VK_LoongArch_CALL36) .Default(VK_LoongArch_Invalid); } diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h index 93251f824103..bd828116d7fa 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h @@ -61,6 +61,7 @@ public: VK_LoongArch_TLS_LD_HI20, VK_LoongArch_TLS_GD_PC_HI20, VK_LoongArch_TLS_GD_HI20, + VK_LoongArch_CALL36, VK_LoongArch_Invalid // Must be the last item. }; diff --git a/llvm/test/CodeGen/LoongArch/code-models.ll b/llvm/test/CodeGen/LoongArch/code-models.ll index c610f645a06a..7c6f46d5e926 100644 --- a/llvm/test/CodeGen/LoongArch/code-models.ll +++ b/llvm/test/CodeGen/LoongArch/code-models.ll @@ -23,8 +23,8 @@ define i32 @call_globaladdress(i32 %a) nounwind { ; MEDIUM: # %bb.0: ; MEDIUM-NEXT: addi.d $sp, $sp, -16 ; MEDIUM-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill -; MEDIUM-NEXT: pcalau12i $ra, %pc_hi20(callee) -; MEDIUM-NEXT: jirl $ra, $ra, %pc_lo12(callee) +; MEDIUM-NEXT: pcaddu18i $ra, %call36(callee) +; MEDIUM-NEXT: jirl $ra, $ra, 0 ; MEDIUM-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; MEDIUM-NEXT: addi.d $sp, $sp, 16 ; MEDIUM-NEXT: ret @@ -68,8 +68,8 @@ define void @call_external_sym(ptr %dst) { ; MEDIUM-NEXT: .cfi_offset 1, -8 ; MEDIUM-NEXT: ori $a2, $zero, 1000 ; MEDIUM-NEXT: move $a1, $zero -; MEDIUM-NEXT: pcalau12i $ra, %pc_hi20(memset) -; MEDIUM-NEXT: jirl $ra, $ra, %pc_lo12(memset) +; MEDIUM-NEXT: pcaddu18i $ra, %call36(memset) +; MEDIUM-NEXT: jirl $ra, $ra, 0 ; MEDIUM-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; MEDIUM-NEXT: addi.d $sp, $sp, 16 ; MEDIUM-NEXT: ret @@ -105,8 +105,8 @@ define i32 @caller_tail(i32 %i) nounwind { ; ; MEDIUM-LABEL: caller_tail: ; MEDIUM: # %bb.0: # %entry -; MEDIUM-NEXT: pcalau12i $a1, %pc_hi20(callee_tail) -; MEDIUM-NEXT: jirl $zero, $a1, %pc_lo12(callee_tail) +; MEDIUM-NEXT: pcaddu18i $a1, %call36(callee_tail) +; MEDIUM-NEXT: jr $a1 ; ; LARGE-LABEL: caller_tail: ; LARGE: # %bb.0: # %entry diff --git a/llvm/test/MC/LoongArch/Basic/Integer/invalid64.s b/llvm/test/MC/LoongArch/Basic/Integer/invalid64.s index acddca9432a6..1c1c658ad440 100644 --- a/llvm/test/MC/LoongArch/Basic/Integer/invalid64.s +++ b/llvm/test/MC/LoongArch/Basic/Integer/invalid64.s @@ -65,7 +65,7 @@ addu16i.d $a0, $a0, 32768 ## simm20 pcaddu18i $a0, 0x80000 -# CHECK: :[[#@LINE-1]]:16: error: immediate must be an integer in the range [-524288, 524287] +# CHECK: :[[#@LINE-1]]:16: error: operand must be a symbol with modifier (e.g. %call36) or an integer in the range [-524288, 524287] ## simm20_lu32id lu32i.d $a0, 0x80000 diff --git a/llvm/test/MC/LoongArch/Macros/macros-call.s b/llvm/test/MC/LoongArch/Macros/macros-call.s new file mode 100644 index 000000000000..a648a3978038 --- /dev/null +++ b/llvm/test/MC/LoongArch/Macros/macros-call.s @@ -0,0 +1,9 @@ +# RUN: llvm-mc --triple=loongarch64 %s | FileCheck %s + +call36 sym_call +# CHECK: pcaddu18i $ra, %call36(sym_call) +# CHECK-NEXT: jirl $ra, $ra, 0 + +tail36 $t0, sym_tail +# CHECK: pcaddu18i $t0, %call36(sym_tail) +# CHECK-NEXT: jr $t0 diff --git a/llvm/test/MC/LoongArch/Relocations/relocations.s b/llvm/test/MC/LoongArch/Relocations/relocations.s index 042cc93470a1..bec71e103893 100644 --- a/llvm/test/MC/LoongArch/Relocations/relocations.s +++ b/llvm/test/MC/LoongArch/Relocations/relocations.s @@ -218,3 +218,8 @@ lu12i.w $t1, %gd_hi20(foo) # RELOC: R_LARCH_TLS_GD_HI20 foo 0x0 # INSTR: lu12i.w $t1, %gd_hi20(foo) # FIXUP: fixup A - offset: 0, value: %gd_hi20(foo), kind: FK_NONE + +pcaddu18i $t1, %call36(foo) +# RELOC: R_LARCH_CALL36 foo 0x0 +# INSTR: pcaddu18i $t1, %call36(foo) +# FIXUP: fixup A - offset: 0, value: %call36(foo), kind: FK_NONE -- GitLab From 3d6fc35b9071009c5ef37f879a12982c6a54db60 Mon Sep 17 00:00:00 2001 From: wanglei Date: Tue, 2 Jan 2024 10:57:40 +0800 Subject: [PATCH 062/728] [LoongArch] Pre-commit test for #76555. NFC --- .../LoongArch/psabi-restricted-scheduling.ll | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll diff --git a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll new file mode 100644 index 000000000000..150a935d7bf8 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll @@ -0,0 +1,172 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc --mtriple=loongarch64 --code-model=medium --post-RA-scheduler=0 < %s \ +; RUN: | FileCheck %s --check-prefix=MEDIUM_NO_SCH +; RUN: llc --mtriple=loongarch64 --code-model=medium --post-RA-scheduler=1 < %s \ +; RUN: | FileCheck %s --check-prefix=MEDIUM_SCH +; RUN: llc --mtriple=loongarch64 --code-model=large --post-RA-scheduler=0 < %s \ +; RUN: | FileCheck %s --check-prefix=LARGE_NO_SCH +; RUN: llc --mtriple=loongarch64 --code-model=large --post-RA-scheduler=1 < %s \ +; RUN: | FileCheck %s --check-prefix=LARGE_SCH + +;; FIXME: According to the description of the psABI v2.30, the code sequences +;; of `PseudoLA*_LARGE` instruction and Medium code model's function call must +;; be adjacent. + +@g = dso_local global i64 zeroinitializer, align 4 +@G = global i64 zeroinitializer, align 4 +@gd = external thread_local global i64 +@ld = external thread_local(localdynamic) global i64 +@ie = external thread_local(initialexec) global i64 + +declare ptr @bar(i64) + +define void @foo() nounwind { +; MEDIUM_NO_SCH-LABEL: foo: +; MEDIUM_NO_SCH: # %bb.0: +; MEDIUM_NO_SCH-NEXT: addi.d $sp, $sp, -16 +; MEDIUM_NO_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill +; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) +; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %got_pc_lo12(G) +; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) +; MEDIUM_NO_SCH-NEXT: addi.d $a0, $a0, %pc_lo12(g) +; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_NO_SCH-NEXT: ori $a0, $zero, 1 +; MEDIUM_NO_SCH-NEXT: pcaddu18i $ra, %call36(bar) +; MEDIUM_NO_SCH-NEXT: jirl $ra, $ra, 0 +; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) +; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(gd) +; MEDIUM_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) +; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ld) +; MEDIUM_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) +; MEDIUM_NO_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ie) +; MEDIUM_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_NO_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload +; MEDIUM_NO_SCH-NEXT: addi.d $sp, $sp, 16 +; MEDIUM_NO_SCH-NEXT: ret +; +; MEDIUM_SCH-LABEL: foo: +; MEDIUM_SCH: # %bb.0: +; MEDIUM_SCH-NEXT: addi.d $sp, $sp, -16 +; MEDIUM_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill +; MEDIUM_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) +; MEDIUM_SCH-NEXT: pcaddu18i $ra, %call36(bar) +; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %got_pc_lo12(G) +; MEDIUM_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) +; MEDIUM_SCH-NEXT: addi.d $a0, $a0, %pc_lo12(g) +; MEDIUM_SCH-NEXT: ld.d $a0, $a0, 0 +; MEDIUM_SCH-NEXT: ori $a0, $zero, 1 +; MEDIUM_SCH-NEXT: jirl $ra, $ra, 0 +; MEDIUM_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) +; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(gd) +; MEDIUM_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) +; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ld) +; MEDIUM_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) +; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(ie) +; MEDIUM_SCH-NEXT: ldx.d $a0, $a0, $tp +; MEDIUM_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload +; MEDIUM_SCH-NEXT: addi.d $sp, $sp, 16 +; MEDIUM_SCH-NEXT: ret +; +; LARGE_NO_SCH-LABEL: foo: +; LARGE_NO_SCH: # %bb.0: +; LARGE_NO_SCH-NEXT: addi.d $sp, $sp, -16 +; LARGE_NO_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill +; LARGE_NO_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) +; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %got_pc_lo12(G) +; LARGE_NO_SCH-NEXT: lu32i.d $a1, %got64_pc_lo20(G) +; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(G) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_NO_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) +; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %pc_lo12(g) +; LARGE_NO_SCH-NEXT: lu32i.d $a1, %pc64_lo20(g) +; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %pc64_hi12(g) +; LARGE_NO_SCH-NEXT: add.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_NO_SCH-NEXT: ori $a0, $zero, 1 +; LARGE_NO_SCH-NEXT: pcalau12i $a1, %got_pc_hi20(bar) +; LARGE_NO_SCH-NEXT: addi.d $ra, $zero, %got_pc_lo12(bar) +; LARGE_NO_SCH-NEXT: lu32i.d $ra, %got64_pc_lo20(bar) +; LARGE_NO_SCH-NEXT: lu52i.d $ra, $ra, %got64_pc_hi12(bar) +; LARGE_NO_SCH-NEXT: ldx.d $ra, $ra, $a1 +; LARGE_NO_SCH-NEXT: jirl $ra, $ra, 0 +; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) +; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(gd) +; LARGE_NO_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(gd) +; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(gd) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) +; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ld) +; LARGE_NO_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ld) +; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ld) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) +; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ie) +; LARGE_NO_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ie) +; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ie) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_NO_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload +; LARGE_NO_SCH-NEXT: addi.d $sp, $sp, 16 +; LARGE_NO_SCH-NEXT: ret +; +; LARGE_SCH-LABEL: foo: +; LARGE_SCH: # %bb.0: +; LARGE_SCH-NEXT: addi.d $sp, $sp, -16 +; LARGE_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill +; LARGE_SCH-NEXT: addi.d $a1, $zero, %got_pc_lo12(G) +; LARGE_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) +; LARGE_SCH-NEXT: addi.d $ra, $zero, %got_pc_lo12(bar) +; LARGE_SCH-NEXT: lu32i.d $a1, %got64_pc_lo20(G) +; LARGE_SCH-NEXT: lu32i.d $ra, %got64_pc_lo20(bar) +; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(G) +; LARGE_SCH-NEXT: lu52i.d $ra, $ra, %got64_pc_hi12(bar) +; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_SCH-NEXT: addi.d $a1, $zero, %pc_lo12(g) +; LARGE_SCH-NEXT: lu32i.d $a1, %pc64_lo20(g) +; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %pc64_hi12(g) +; LARGE_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) +; LARGE_SCH-NEXT: add.d $a0, $a1, $a0 +; LARGE_SCH-NEXT: pcalau12i $a1, %got_pc_hi20(bar) +; LARGE_SCH-NEXT: ld.d $a0, $a0, 0 +; LARGE_SCH-NEXT: ldx.d $ra, $ra, $a1 +; LARGE_SCH-NEXT: ori $a0, $zero, 1 +; LARGE_SCH-NEXT: jirl $ra, $ra, 0 +; LARGE_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(gd) +; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) +; LARGE_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(gd) +; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(gd) +; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ld) +; LARGE_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ld) +; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ld) +; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) +; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ie) +; LARGE_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ie) +; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ie) +; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) +; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp +; LARGE_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload +; LARGE_SCH-NEXT: addi.d $sp, $sp, 16 +; LARGE_SCH-NEXT: ret + %V = load volatile i64, ptr @G + %v = load volatile i64, ptr @g + call void @bar(i64 1) + %v_gd = load volatile i64, ptr @gd + %v_ld = load volatile i64, ptr @ld + %v_ie = load volatile i64, ptr @ie + ret void +} -- GitLab From c56a5e895a96fec4292e9333d998cfa88770432a Mon Sep 17 00:00:00 2001 From: wanglei Date: Tue, 2 Jan 2024 10:57:15 +0800 Subject: [PATCH 063/728] [LoongArch] Reimplement the expansion of PseudoLA*_LARGE instructions (#76555) According to the description of the psABI v2.30: https://github.com/loongson/la-abi-specs/releases/tag/v2.30, moved the expansion of relevant pseudo-instructions from `LoongArchPreRAExpandPseudo` pass to `LoongArchExpandPseudo` pass, to ensure that the code sequences of `PseudoLA*_LARGE` instructions and Medium code model's function call are not scheduled. --- .../LoongArch/LoongArchExpandPseudoInsts.cpp | 519 +++++++++--------- .../LoongArch/LoongArchISelLowering.cpp | 24 +- .../Target/LoongArch/LoongArchISelLowering.h | 4 + .../Target/LoongArch/LoongArchInstrInfo.td | 83 ++- llvm/test/CodeGen/LoongArch/code-models.ll | 36 +- llvm/test/CodeGen/LoongArch/expand-call.ll | 2 +- llvm/test/CodeGen/LoongArch/global-address.ll | 32 +- .../LoongArch/psabi-restricted-scheduling.ll | 102 ++-- llvm/test/CodeGen/LoongArch/tls-models.ll | 68 +-- 9 files changed, 487 insertions(+), 383 deletions(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp index 8eda2dcc1633..f977f176066a 100644 --- a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp @@ -62,43 +62,24 @@ private: MachineBasicBlock::iterator &NextMBBI, unsigned FlagsHi, unsigned SecondOpcode, unsigned FlagsLo); - bool expandLargeAddressLoad(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - unsigned LastOpcode, unsigned IdentifyingMO); - bool expandLargeAddressLoad(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - unsigned LastOpcode, unsigned IdentifyingMO, - const MachineOperand &Symbol, Register DestReg, - bool EraseFromParent); bool expandLoadAddressPcrel(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - bool Large = false); + MachineBasicBlock::iterator &NextMBBI); bool expandLoadAddressGot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - bool Large = false); + MachineBasicBlock::iterator &NextMBBI); bool expandLoadAddressTLSLE(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NextMBBI); bool expandLoadAddressTLSIE(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - bool Large = false); + MachineBasicBlock::iterator &NextMBBI); bool expandLoadAddressTLSLD(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - bool Large = false); + MachineBasicBlock::iterator &NextMBBI); bool expandLoadAddressTLSGD(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - bool Large = false); - bool expandFunctionCALL(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, - bool IsTailCall); + MachineBasicBlock::iterator &NextMBBI); }; char LoongArchPreRAExpandPseudo::ID = 0; @@ -131,30 +112,16 @@ bool LoongArchPreRAExpandPseudo::expandMI( switch (MBBI->getOpcode()) { case LoongArch::PseudoLA_PCREL: return expandLoadAddressPcrel(MBB, MBBI, NextMBBI); - case LoongArch::PseudoLA_PCREL_LARGE: - return expandLoadAddressPcrel(MBB, MBBI, NextMBBI, /*Large=*/true); case LoongArch::PseudoLA_GOT: return expandLoadAddressGot(MBB, MBBI, NextMBBI); - case LoongArch::PseudoLA_GOT_LARGE: - return expandLoadAddressGot(MBB, MBBI, NextMBBI, /*Large=*/true); case LoongArch::PseudoLA_TLS_LE: return expandLoadAddressTLSLE(MBB, MBBI, NextMBBI); case LoongArch::PseudoLA_TLS_IE: return expandLoadAddressTLSIE(MBB, MBBI, NextMBBI); - case LoongArch::PseudoLA_TLS_IE_LARGE: - return expandLoadAddressTLSIE(MBB, MBBI, NextMBBI, /*Large=*/true); case LoongArch::PseudoLA_TLS_LD: return expandLoadAddressTLSLD(MBB, MBBI, NextMBBI); - case LoongArch::PseudoLA_TLS_LD_LARGE: - return expandLoadAddressTLSLD(MBB, MBBI, NextMBBI, /*Large=*/true); case LoongArch::PseudoLA_TLS_GD: return expandLoadAddressTLSGD(MBB, MBBI, NextMBBI); - case LoongArch::PseudoLA_TLS_GD_LARGE: - return expandLoadAddressTLSGD(MBB, MBBI, NextMBBI, /*Large=*/true); - case LoongArch::PseudoCALL: - return expandFunctionCALL(MBB, MBBI, NextMBBI, /*IsTailCall=*/false); - case LoongArch::PseudoTAIL: - return expandFunctionCALL(MBB, MBBI, NextMBBI, /*IsTailCall=*/true); } return false; } @@ -187,118 +154,9 @@ bool LoongArchPreRAExpandPseudo::expandPcalau12iInstPair( return true; } -bool LoongArchPreRAExpandPseudo::expandLargeAddressLoad( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, unsigned LastOpcode, - unsigned IdentifyingMO) { - MachineInstr &MI = *MBBI; - return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LastOpcode, IdentifyingMO, - MI.getOperand(2), MI.getOperand(0).getReg(), - true); -} - -bool LoongArchPreRAExpandPseudo::expandLargeAddressLoad( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, unsigned LastOpcode, - unsigned IdentifyingMO, const MachineOperand &Symbol, Register DestReg, - bool EraseFromParent) { - // Code Sequence: - // - // Part1: pcalau12i $scratch, %MO1(sym) - // Part0: addi.d $dest, $zero, %MO0(sym) - // Part2: lu32i.d $dest, %MO2(sym) - // Part3: lu52i.d $dest, $dest, %MO3(sym) - // Fin: LastOpcode $dest, $dest, $scratch - - unsigned MO0, MO1, MO2, MO3; - switch (IdentifyingMO) { - default: - llvm_unreachable("unsupported identifying MO"); - case LoongArchII::MO_PCREL_LO: - MO0 = IdentifyingMO; - MO1 = LoongArchII::MO_PCREL_HI; - MO2 = LoongArchII::MO_PCREL64_LO; - MO3 = LoongArchII::MO_PCREL64_HI; - break; - case LoongArchII::MO_GOT_PC_HI: - case LoongArchII::MO_LD_PC_HI: - case LoongArchII::MO_GD_PC_HI: - // These cases relocate just like the GOT case, except for Part1. - MO0 = LoongArchII::MO_GOT_PC_LO; - MO1 = IdentifyingMO; - MO2 = LoongArchII::MO_GOT_PC64_LO; - MO3 = LoongArchII::MO_GOT_PC64_HI; - break; - case LoongArchII::MO_IE_PC_LO: - MO0 = IdentifyingMO; - MO1 = LoongArchII::MO_IE_PC_HI; - MO2 = LoongArchII::MO_IE_PC64_LO; - MO3 = LoongArchII::MO_IE_PC64_HI; - break; - } - - MachineFunction *MF = MBB.getParent(); - MachineInstr &MI = *MBBI; - DebugLoc DL = MI.getDebugLoc(); - - assert(MF->getSubtarget().is64Bit() && - "Large code model requires LA64"); - - Register TmpPart1 = - MF->getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass); - Register TmpPart0 = - DestReg.isVirtual() - ? MF->getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass) - : DestReg; - Register TmpParts02 = - DestReg.isVirtual() - ? MF->getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass) - : DestReg; - Register TmpParts023 = - DestReg.isVirtual() - ? MF->getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass) - : DestReg; - - auto Part1 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::PCALAU12I), TmpPart1); - auto Part0 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::ADDI_D), TmpPart0) - .addReg(LoongArch::R0); - auto Part2 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::LU32I_D), TmpParts02) - // "rj" is needed due to InstrInfo pattern requirement. - .addReg(TmpPart0, RegState::Kill); - auto Part3 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::LU52I_D), TmpParts023) - .addReg(TmpParts02, RegState::Kill); - BuildMI(MBB, MBBI, DL, TII->get(LastOpcode), DestReg) - .addReg(TmpParts023) - .addReg(TmpPart1, RegState::Kill); - - if (Symbol.getType() == MachineOperand::MO_ExternalSymbol) { - const char *SymName = Symbol.getSymbolName(); - Part0.addExternalSymbol(SymName, MO0); - Part1.addExternalSymbol(SymName, MO1); - Part2.addExternalSymbol(SymName, MO2); - Part3.addExternalSymbol(SymName, MO3); - } else { - Part0.addDisp(Symbol, 0, MO0); - Part1.addDisp(Symbol, 0, MO1); - Part2.addDisp(Symbol, 0, MO2); - Part3.addDisp(Symbol, 0, MO3); - } - - if (EraseFromParent) - MI.eraseFromParent(); - - return true; -} - bool LoongArchPreRAExpandPseudo::expandLoadAddressPcrel( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, bool Large) { - if (Large) - // Emit the 5-insn large address load sequence with the `%pc` family of - // relocs. - return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::ADD_D, - LoongArchII::MO_PCREL_LO); - + MachineBasicBlock::iterator &NextMBBI) { // Code Sequence: // pcalau12i $rd, %pc_hi20(sym) // addi.w/d $rd, $rd, %pc_lo12(sym) @@ -311,13 +169,7 @@ bool LoongArchPreRAExpandPseudo::expandLoadAddressPcrel( bool LoongArchPreRAExpandPseudo::expandLoadAddressGot( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, bool Large) { - if (Large) - // Emit the 5-insn large address load sequence with the `%got_pc` family - // of relocs, loading the result from GOT with `ldx.d` in the end. - return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::LDX_D, - LoongArchII::MO_GOT_PC_HI); - + MachineBasicBlock::iterator &NextMBBI) { // Code Sequence: // pcalau12i $rd, %got_pc_hi20(sym) // ld.w/d $rd, $rd, %got_pc_lo12(sym) @@ -378,13 +230,7 @@ bool LoongArchPreRAExpandPseudo::expandLoadAddressTLSLE( bool LoongArchPreRAExpandPseudo::expandLoadAddressTLSIE( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, bool Large) { - if (Large) - // Emit the 5-insn large address load sequence with the `%ie_pc` family - // of relocs, loading the result with `ldx.d` in the end. - return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::LDX_D, - LoongArchII::MO_IE_PC_LO); - + MachineBasicBlock::iterator &NextMBBI) { // Code Sequence: // pcalau12i $rd, %ie_pc_hi20(sym) // ld.w/d $rd, $rd, %ie_pc_lo12(sym) @@ -397,13 +243,7 @@ bool LoongArchPreRAExpandPseudo::expandLoadAddressTLSIE( bool LoongArchPreRAExpandPseudo::expandLoadAddressTLSLD( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, bool Large) { - if (Large) - // Emit the 5-insn large address load sequence with the `%got_pc` family - // of relocs, with the `pcalau12i` insn relocated with `%ld_pc_hi20`. - return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::ADD_D, - LoongArchII::MO_LD_PC_HI); - + MachineBasicBlock::iterator &NextMBBI) { // Code Sequence: // pcalau12i $rd, %ld_pc_hi20(sym) // addi.w/d $rd, $rd, %got_pc_lo12(sym) @@ -416,13 +256,7 @@ bool LoongArchPreRAExpandPseudo::expandLoadAddressTLSLD( bool LoongArchPreRAExpandPseudo::expandLoadAddressTLSGD( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, bool Large) { - if (Large) - // Emit the 5-insn large address load sequence with the `%got_pc` family - // of relocs, with the `pcalau12i` insn relocated with `%gd_pc_hi20`. - return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::ADD_D, - LoongArchII::MO_GD_PC_HI); - + MachineBasicBlock::iterator &NextMBBI) { // Code Sequence: // pcalau12i $rd, %gd_pc_hi20(sym) // addi.w/d $rd, $rd, %got_pc_lo12(sym) @@ -433,85 +267,6 @@ bool LoongArchPreRAExpandPseudo::expandLoadAddressTLSGD( SecondOpcode, LoongArchII::MO_GOT_PC_LO); } -bool LoongArchPreRAExpandPseudo::expandFunctionCALL( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, - MachineBasicBlock::iterator &NextMBBI, bool IsTailCall) { - MachineFunction *MF = MBB.getParent(); - MachineInstr &MI = *MBBI; - DebugLoc DL = MI.getDebugLoc(); - const MachineOperand &Func = MI.getOperand(0); - MachineInstrBuilder CALL; - unsigned Opcode; - - switch (MF->getTarget().getCodeModel()) { - default: - report_fatal_error("Unsupported code model"); - break; - case CodeModel::Small: { - // CALL: - // bl func - // TAIL: - // b func - Opcode = IsTailCall ? LoongArch::PseudoB_TAIL : LoongArch::BL; - CALL = BuildMI(MBB, MBBI, DL, TII->get(Opcode)).add(Func); - break; - } - case CodeModel::Medium: { - // CALL: - // pcaddu18i $ra, %call36(func) - // jirl $ra, $ra, 0 - // TAIL: - // pcaddu18i $scratch, %call36(func) - // jirl $r0, $scratch, 0 - Opcode = - IsTailCall ? LoongArch::PseudoJIRL_TAIL : LoongArch::PseudoJIRL_CALL; - Register ScratchReg = - IsTailCall - ? MF->getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass) - : LoongArch::R1; - MachineInstrBuilder MIB = - BuildMI(MBB, MBBI, DL, TII->get(LoongArch::PCADDU18I), ScratchReg); - - CALL = - BuildMI(MBB, MBBI, DL, TII->get(Opcode)).addReg(ScratchReg).addImm(0); - - if (Func.isSymbol()) - MIB.addExternalSymbol(Func.getSymbolName(), LoongArchII::MO_CALL36); - else - MIB.addDisp(Func, 0, LoongArchII::MO_CALL36); - break; - } - case CodeModel::Large: { - // Emit the 5-insn large address load sequence, either directly or - // indirectly in case of going through the GOT, then JIRL_TAIL or - // JIRL_CALL to $addr. - Opcode = - IsTailCall ? LoongArch::PseudoJIRL_TAIL : LoongArch::PseudoJIRL_CALL; - Register AddrReg = - IsTailCall - ? MF->getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass) - : LoongArch::R1; - - bool UseGOT = Func.isGlobal() && !Func.getGlobal()->isDSOLocal(); - unsigned MO = UseGOT ? LoongArchII::MO_GOT_PC_HI : LoongArchII::MO_PCREL_LO; - unsigned LAOpcode = UseGOT ? LoongArch::LDX_D : LoongArch::ADD_D; - expandLargeAddressLoad(MBB, MBBI, NextMBBI, LAOpcode, MO, Func, AddrReg, - false); - CALL = BuildMI(MBB, MBBI, DL, TII->get(Opcode)).addReg(AddrReg).addImm(0); - break; - } - } - - // Transfer implicit operands. - CALL.copyImplicitOps(MI); - - // Transfer MI flags. - CALL.setMIFlags(MI.getFlags()); - - MI.eraseFromParent(); - return true; -} - class LoongArchExpandPseudo : public MachineFunctionPass { public: const LoongArchInstrInfo *TII; @@ -533,6 +288,35 @@ private: MachineBasicBlock::iterator &NextMBBI); bool expandCopyCFR(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NextMBBI); + bool expandLargeAddressLoad(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI, + unsigned LastOpcode, unsigned IdentifyingMO); + bool expandLargeAddressLoad(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI, + unsigned LastOpcode, unsigned IdentifyingMO, + const MachineOperand &Symbol, Register DestReg, + bool EraseFromParent); + bool expandLoadAddressPcrelLarge(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + bool expandLoadAddressGotLarge(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + bool expandLoadAddressTLSIELarge(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + bool expandLoadAddressTLSLDLarge(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + bool expandLoadAddressTLSGDLarge(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI); + bool expandFunctionCALL(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI, + bool IsTailCall); }; char LoongArchExpandPseudo::ID = 0; @@ -567,6 +351,24 @@ bool LoongArchExpandPseudo::expandMI(MachineBasicBlock &MBB, switch (MBBI->getOpcode()) { case LoongArch::PseudoCopyCFR: return expandCopyCFR(MBB, MBBI, NextMBBI); + case LoongArch::PseudoLA_PCREL_LARGE: + return expandLoadAddressPcrelLarge(MBB, MBBI, NextMBBI); + case LoongArch::PseudoLA_GOT_LARGE: + return expandLoadAddressGotLarge(MBB, MBBI, NextMBBI); + case LoongArch::PseudoLA_TLS_IE_LARGE: + return expandLoadAddressTLSIELarge(MBB, MBBI, NextMBBI); + case LoongArch::PseudoLA_TLS_LD_LARGE: + return expandLoadAddressTLSLDLarge(MBB, MBBI, NextMBBI); + case LoongArch::PseudoLA_TLS_GD_LARGE: + return expandLoadAddressTLSGDLarge(MBB, MBBI, NextMBBI); + case LoongArch::PseudoCALL: + case LoongArch::PseudoCALL_MEDIUM: + case LoongArch::PseudoCALL_LARGE: + return expandFunctionCALL(MBB, MBBI, NextMBBI, /*IsTailCall=*/false); + case LoongArch::PseudoTAIL: + case LoongArch::PseudoTAIL_MEDIUM: + case LoongArch::PseudoTAIL_LARGE: + return expandFunctionCALL(MBB, MBBI, NextMBBI, /*IsTailCall=*/true); } return false; @@ -625,6 +427,213 @@ bool LoongArchExpandPseudo::expandCopyCFR( return true; } +bool LoongArchExpandPseudo::expandLargeAddressLoad( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI, unsigned LastOpcode, + unsigned IdentifyingMO) { + MachineInstr &MI = *MBBI; + return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LastOpcode, IdentifyingMO, + MI.getOperand(2), MI.getOperand(0).getReg(), + true); +} + +bool LoongArchExpandPseudo::expandLargeAddressLoad( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI, unsigned LastOpcode, + unsigned IdentifyingMO, const MachineOperand &Symbol, Register DestReg, + bool EraseFromParent) { + // Code Sequence: + // + // Part1: pcalau12i $dst, %MO1(sym) + // Part0: addi.d $t8, $zero, %MO0(sym) + // Part2: lu32i.d $t8, %MO2(sym) + // Part3: lu52i.d $t8, $t8, %MO3(sym) + // Fin: LastOpcode $dst, $t8, $dst + + unsigned MO0, MO1, MO2, MO3; + switch (IdentifyingMO) { + default: + llvm_unreachable("unsupported identifying MO"); + case LoongArchII::MO_PCREL_LO: + MO0 = IdentifyingMO; + MO1 = LoongArchII::MO_PCREL_HI; + MO2 = LoongArchII::MO_PCREL64_LO; + MO3 = LoongArchII::MO_PCREL64_HI; + break; + case LoongArchII::MO_GOT_PC_HI: + case LoongArchII::MO_LD_PC_HI: + case LoongArchII::MO_GD_PC_HI: + // These cases relocate just like the GOT case, except for Part1. + MO0 = LoongArchII::MO_GOT_PC_LO; + MO1 = IdentifyingMO; + MO2 = LoongArchII::MO_GOT_PC64_LO; + MO3 = LoongArchII::MO_GOT_PC64_HI; + break; + case LoongArchII::MO_IE_PC_LO: + MO0 = IdentifyingMO; + MO1 = LoongArchII::MO_IE_PC_HI; + MO2 = LoongArchII::MO_IE_PC64_LO; + MO3 = LoongArchII::MO_IE_PC64_HI; + break; + } + + MachineFunction *MF = MBB.getParent(); + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + Register ScratchReg = LoongArch::R20; // $t8 + + assert(MF->getSubtarget().is64Bit() && + "Large code model requires LA64"); + + auto Part1 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::PCALAU12I), DestReg); + auto Part0 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::ADDI_D), ScratchReg) + .addReg(LoongArch::R0); + auto Part2 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::LU32I_D), ScratchReg) + // "rj" is needed due to InstrInfo pattern requirement. + .addReg(ScratchReg); + auto Part3 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::LU52I_D), ScratchReg) + .addReg(ScratchReg); + BuildMI(MBB, MBBI, DL, TII->get(LastOpcode), DestReg) + .addReg(ScratchReg) + .addReg(DestReg); + + if (Symbol.getType() == MachineOperand::MO_ExternalSymbol) { + const char *SymName = Symbol.getSymbolName(); + Part0.addExternalSymbol(SymName, MO0); + Part1.addExternalSymbol(SymName, MO1); + Part2.addExternalSymbol(SymName, MO2); + Part3.addExternalSymbol(SymName, MO3); + } else { + Part0.addDisp(Symbol, 0, MO0); + Part1.addDisp(Symbol, 0, MO1); + Part2.addDisp(Symbol, 0, MO2); + Part3.addDisp(Symbol, 0, MO3); + } + + if (EraseFromParent) + MI.eraseFromParent(); + + return true; +} + +bool LoongArchExpandPseudo::expandLoadAddressPcrelLarge( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + // Emit the 5-insn large address load sequence with the `%pc` family of + // relocs. + return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::ADD_D, + LoongArchII::MO_PCREL_LO); +} + +bool LoongArchExpandPseudo::expandLoadAddressGotLarge( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + // Emit the 5-insn large address load sequence with the `%got_pc` family + // of relocs, loading the result from GOT with `ldx.d` in the end. + return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::LDX_D, + LoongArchII::MO_GOT_PC_HI); +} + +bool LoongArchExpandPseudo::expandLoadAddressTLSIELarge( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + // Emit the 5-insn large address load sequence with the `%ie_pc` family + // of relocs, loading the result with `ldx.d` in the end. + return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::LDX_D, + LoongArchII::MO_IE_PC_LO); +} + +bool LoongArchExpandPseudo::expandLoadAddressTLSLDLarge( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + // Emit the 5-insn large address load sequence with the `%got_pc` family + // of relocs, with the `pcalau12i` insn relocated with `%ld_pc_hi20`. + return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::ADD_D, + LoongArchII::MO_LD_PC_HI); +} + +bool LoongArchExpandPseudo::expandLoadAddressTLSGDLarge( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI) { + // Emit the 5-insn large address load sequence with the `%got_pc` family + // of relocs, with the `pcalau12i` insn relocated with `%gd_pc_hi20`. + return expandLargeAddressLoad(MBB, MBBI, NextMBBI, LoongArch::ADD_D, + LoongArchII::MO_GD_PC_HI); +} + +bool LoongArchExpandPseudo::expandFunctionCALL( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MachineBasicBlock::iterator &NextMBBI, bool IsTailCall) { + MachineFunction *MF = MBB.getParent(); + MachineInstr &MI = *MBBI; + DebugLoc DL = MI.getDebugLoc(); + const MachineOperand &Func = MI.getOperand(0); + MachineInstrBuilder CALL; + unsigned Opcode; + + switch (MF->getTarget().getCodeModel()) { + default: + report_fatal_error("Unsupported code model"); + break; + case CodeModel::Small: { + // CALL: + // bl func + // TAIL: + // b func + Opcode = IsTailCall ? LoongArch::PseudoB_TAIL : LoongArch::BL; + CALL = BuildMI(MBB, MBBI, DL, TII->get(Opcode)).add(Func); + break; + } + case CodeModel::Medium: { + // CALL: + // pcaddu18i $ra, %call36(func) + // jirl $ra, $ra, 0 + // TAIL: + // pcaddu18i $t8, %call36(func) + // jr $t8 + Opcode = + IsTailCall ? LoongArch::PseudoJIRL_TAIL : LoongArch::PseudoJIRL_CALL; + Register ScratchReg = IsTailCall ? LoongArch::R20 : LoongArch::R1; + MachineInstrBuilder MIB = + BuildMI(MBB, MBBI, DL, TII->get(LoongArch::PCADDU18I), ScratchReg); + + CALL = + BuildMI(MBB, MBBI, DL, TII->get(Opcode)).addReg(ScratchReg).addImm(0); + + if (Func.isSymbol()) + MIB.addExternalSymbol(Func.getSymbolName(), LoongArchII::MO_CALL36); + else + MIB.addDisp(Func, 0, LoongArchII::MO_CALL36); + break; + } + case CodeModel::Large: { + // Emit the 5-insn large address load sequence, either directly or + // indirectly in case of going through the GOT, then JIRL_TAIL or + // JIRL_CALL to $addr. + Opcode = + IsTailCall ? LoongArch::PseudoJIRL_TAIL : LoongArch::PseudoJIRL_CALL; + Register AddrReg = IsTailCall ? LoongArch::R19 : LoongArch::R1; + + bool UseGOT = Func.isGlobal() && !Func.getGlobal()->isDSOLocal(); + unsigned MO = UseGOT ? LoongArchII::MO_GOT_PC_HI : LoongArchII::MO_PCREL_LO; + unsigned LAOpcode = UseGOT ? LoongArch::LDX_D : LoongArch::ADD_D; + expandLargeAddressLoad(MBB, MBBI, NextMBBI, LAOpcode, MO, Func, AddrReg, + false); + CALL = BuildMI(MBB, MBBI, DL, TII->get(Opcode)).addReg(AddrReg).addImm(0); + break; + } + } + + // Transfer implicit operands. + CALL.copyImplicitOps(MI); + + // Transfer MI flags. + CALL.setMIFlags(MI.getFlags()); + + MI.eraseFromParent(); + return true; +} + } // end namespace INITIALIZE_PASS(LoongArchPreRAExpandPseudo, "loongarch-prera-expand-pseudo", diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index e14bbadf9ed2..2cfb2c1d7811 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -3381,8 +3381,12 @@ const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const { // TODO: Add more target-dependent nodes later. NODE_NAME_CASE(CALL) + NODE_NAME_CASE(CALL_MEDIUM) + NODE_NAME_CASE(CALL_LARGE) NODE_NAME_CASE(RET) NODE_NAME_CASE(TAIL) + NODE_NAME_CASE(TAIL_MEDIUM) + NODE_NAME_CASE(TAIL_LARGE) NODE_NAME_CASE(SLL_W) NODE_NAME_CASE(SRA_W) NODE_NAME_CASE(SRL_W) @@ -4240,15 +4244,31 @@ LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI, // Emit the call. SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); + unsigned Op; + switch (DAG.getTarget().getCodeModel()) { + default: + report_fatal_error("Unsupported code model"); + case CodeModel::Small: + Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL; + break; + case CodeModel::Medium: + assert(Subtarget.is64Bit() && "Medium code model requires LA64"); + Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM; + break; + case CodeModel::Large: + assert(Subtarget.is64Bit() && "Large code model requires LA64"); + Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE; + break; + } if (IsTailCall) { MF.getFrameInfo().setHasTailCall(); - SDValue Ret = DAG.getNode(LoongArchISD::TAIL, DL, NodeTys, Ops); + SDValue Ret = DAG.getNode(Op, DL, NodeTys, Ops); DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge); return Ret; } - Chain = DAG.getNode(LoongArchISD::CALL, DL, NodeTys, Ops); + Chain = DAG.getNode(Op, DL, NodeTys, Ops); DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); Glue = Chain.getValue(1); diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index 6f8878f9ccd5..2875aa82e424 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -27,8 +27,12 @@ enum NodeType : unsigned { // TODO: add more LoongArchISDs CALL, + CALL_MEDIUM, + CALL_LARGE, RET, TAIL, + TAIL_MEDIUM, + TAIL_LARGE, // 32-bit shifts, directly matching the semantics of the named LoongArch // instructions. diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td index 21555f885630..78074c012876 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td @@ -69,6 +69,18 @@ def loongarch_ret : SDNode<"LoongArchISD::RET", SDTNone, def loongarch_tail : SDNode<"LoongArchISD::TAIL", SDT_LoongArchCall, [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, SDNPVariadic]>; +def loongarch_call_medium : SDNode<"LoongArchISD::CALL_MEDIUM", SDT_LoongArchCall, + [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, + SDNPVariadic]>; +def loongarch_tail_medium : SDNode<"LoongArchISD::TAIL_MEDIUM", SDT_LoongArchCall, + [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, + SDNPVariadic]>; +def loongarch_call_large : SDNode<"LoongArchISD::CALL_LARGE", SDT_LoongArchCall, + [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, + SDNPVariadic]>; +def loongarch_tail_large : SDNode<"LoongArchISD::TAIL_LARGE", SDT_LoongArchCall, + [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, + SDNPVariadic]>; def loongarch_sll_w : SDNode<"LoongArchISD::SLL_W", SDT_LoongArchIntBinOpW>; def loongarch_sra_w : SDNode<"LoongArchISD::SRA_W", SDT_LoongArchIntBinOpW>; def loongarch_srl_w : SDNode<"LoongArchISD::SRL_W", SDT_LoongArchIntBinOpW>; @@ -1399,16 +1411,43 @@ def : Pat<(brind GPR:$rj), (PseudoBRIND GPR:$rj, 0)>; def : Pat<(brind (add GPR:$rj, simm16_lsl2:$imm16)), (PseudoBRIND GPR:$rj, simm16_lsl2:$imm16)>; +// Function call with 'Small' code model. let isCall = 1, Defs = [R1] in def PseudoCALL : Pseudo<(outs), (ins bare_symbol:$func)>; def : Pat<(loongarch_call tglobaladdr:$func), (PseudoCALL tglobaladdr:$func)>; def : Pat<(loongarch_call texternalsym:$func), (PseudoCALL texternalsym:$func)>; +// Function call with 'Medium' code model. +let isCall = 1, Defs = [R1, R20], Size = 8 in +def PseudoCALL_MEDIUM : Pseudo<(outs), (ins bare_symbol:$func)>; + +let Predicates = [IsLA64] in { +def : Pat<(loongarch_call_medium tglobaladdr:$func), + (PseudoCALL_MEDIUM tglobaladdr:$func)>; +def : Pat<(loongarch_call_medium texternalsym:$func), + (PseudoCALL_MEDIUM texternalsym:$func)>; +} // Predicates = [IsLA64] + +// Function call with 'Large' code model. +let isCall = 1, Defs = [R1, R20], Size = 24 in +def PseudoCALL_LARGE: Pseudo<(outs), (ins bare_symbol:$func)>; + +let Predicates = [IsLA64] in { +def : Pat<(loongarch_call_large tglobaladdr:$func), + (PseudoCALL_LARGE tglobaladdr:$func)>; +def : Pat<(loongarch_call_large texternalsym:$func), + (PseudoCALL_LARGE texternalsym:$func)>; +} // Predicates = [IsLA64] + let isCall = 1, Defs = [R1] in def PseudoCALLIndirect : Pseudo<(outs), (ins GPR:$rj), [(loongarch_call GPR:$rj)]>, PseudoInstExpansion<(JIRL R1, GPR:$rj, 0)>; +let Predicates = [IsLA64] in { +def : Pat<(loongarch_call_medium GPR:$rj), (PseudoCALLIndirect GPR:$rj)>; +def : Pat<(loongarch_call_large GPR:$rj), (PseudoCALLIndirect GPR:$rj)>; +} let isCall = 1, hasSideEffects = 0, mayStore = 0, mayLoad = 0, Defs = [R1] in def PseudoJIRL_CALL : Pseudo<(outs), (ins GPR:$rj, simm16_lsl2:$imm16)>, @@ -1419,6 +1458,7 @@ let isBarrier = 1, isReturn = 1, isTerminator = 1 in def PseudoRET : Pseudo<(outs), (ins), [(loongarch_ret)]>, PseudoInstExpansion<(JIRL R0, R1, 0)>; +// Tail call with 'Small' code model. let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [R3] in def PseudoTAIL : Pseudo<(outs), (ins bare_symbol:$dst)>; @@ -1427,10 +1467,38 @@ def : Pat<(loongarch_tail (iPTR tglobaladdr:$dst)), def : Pat<(loongarch_tail (iPTR texternalsym:$dst)), (PseudoTAIL texternalsym:$dst)>; +// Tail call with 'Medium' code model. +let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, + Uses = [R3], Defs = [R20], Size = 8 in +def PseudoTAIL_MEDIUM : Pseudo<(outs), (ins bare_symbol:$dst)>; + +let Predicates = [IsLA64] in { +def : Pat<(loongarch_tail_medium (iPTR tglobaladdr:$dst)), + (PseudoTAIL_MEDIUM tglobaladdr:$dst)>; +def : Pat<(loongarch_tail_medium (iPTR texternalsym:$dst)), + (PseudoTAIL_MEDIUM texternalsym:$dst)>; +} // Predicates = [IsLA64] + +// Tail call with 'Large' code model. +let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, + Uses = [R3], Defs = [R19, R20], Size = 24 in +def PseudoTAIL_LARGE : Pseudo<(outs), (ins bare_symbol:$dst)>; + +let Predicates = [IsLA64] in { +def : Pat<(loongarch_tail_large (iPTR tglobaladdr:$dst)), + (PseudoTAIL_LARGE tglobaladdr:$dst)>; +def : Pat<(loongarch_tail_large (iPTR texternalsym:$dst)), + (PseudoTAIL_LARGE texternalsym:$dst)>; +} // Predicates = [IsLA64] + let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [R3] in def PseudoTAILIndirect : Pseudo<(outs), (ins GPRT:$rj), [(loongarch_tail GPRT:$rj)]>, PseudoInstExpansion<(JIRL R0, GPR:$rj, 0)>; +let Predicates = [IsLA64] in { +def : Pat<(loongarch_tail_medium GPR:$rj), (PseudoTAILIndirect GPR:$rj)>; +def : Pat<(loongarch_tail_large GPR:$rj), (PseudoTAILIndirect GPR:$rj)>; +} let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, hasSideEffects = 0, mayStore = 0, mayLoad = 0, Uses = [R3] in @@ -1468,6 +1536,7 @@ def PseudoLA_ABS_LARGE : Pseudo<(outs GPR:$dst), "la.abs", "$dst, $src">; def PseudoLA_PCREL : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], "la.pcrel", "$dst, $src">; +let Defs = [R20], Size = 20 in def PseudoLA_PCREL_LARGE : Pseudo<(outs GPR:$dst), (ins GPR:$tmp, bare_symbol:$src), [], "la.pcrel", "$dst, $tmp, $src">, @@ -1479,28 +1548,30 @@ let hasSideEffects = 0, mayLoad = 1, mayStore = 0, isCodeGenOnly = 0, isAsmParserOnly = 1 in { def PseudoLA_GOT : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], "la.got", "$dst, $src">; +def PseudoLA_TLS_IE : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], + "la.tls.ie", "$dst, $src">; +def PseudoLA_TLS_LD : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], + "la.tls.ld", "$dst, $src">; +def PseudoLA_TLS_GD : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], + "la.tls.gd", "$dst, $src">; +let Defs = [R20], Size = 20 in { def PseudoLA_GOT_LARGE : Pseudo<(outs GPR:$dst), (ins GPR:$tmp, bare_symbol:$src), [], "la.got", "$dst, $tmp, $src">, Requires<[IsLA64]>; -def PseudoLA_TLS_IE : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], - "la.tls.ie", "$dst, $src">; def PseudoLA_TLS_IE_LARGE : Pseudo<(outs GPR:$dst), (ins GPR:$tmp, bare_symbol:$src), [], "la.tls.ie", "$dst, $tmp, $src">, Requires<[IsLA64]>; -def PseudoLA_TLS_LD : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], - "la.tls.ld", "$dst, $src">; def PseudoLA_TLS_LD_LARGE : Pseudo<(outs GPR:$dst), (ins GPR:$tmp, bare_symbol:$src), [], "la.tls.ld", "$dst, $tmp, $src">, Requires<[IsLA64]>; -def PseudoLA_TLS_GD : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [], - "la.tls.gd", "$dst, $src">; def PseudoLA_TLS_GD_LARGE : Pseudo<(outs GPR:$dst), (ins GPR:$tmp, bare_symbol:$src), [], "la.tls.gd", "$dst, $tmp, $src">, Requires<[IsLA64]>; +} // Defs = [R20], Size = 20 } // Load address inst alias: "la", "la.global" and "la.local". diff --git a/llvm/test/CodeGen/LoongArch/code-models.ll b/llvm/test/CodeGen/LoongArch/code-models.ll index 7c6f46d5e926..f93c31670928 100644 --- a/llvm/test/CodeGen/LoongArch/code-models.ll +++ b/llvm/test/CodeGen/LoongArch/code-models.ll @@ -33,11 +33,11 @@ define i32 @call_globaladdress(i32 %a) nounwind { ; LARGE: # %bb.0: ; LARGE-NEXT: addi.d $sp, $sp, -16 ; LARGE-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill -; LARGE-NEXT: pcalau12i $a1, %got_pc_hi20(callee) -; LARGE-NEXT: addi.d $ra, $zero, %got_pc_lo12(callee) -; LARGE-NEXT: lu32i.d $ra, %got64_pc_lo20(callee) -; LARGE-NEXT: lu52i.d $ra, $ra, %got64_pc_hi12(callee) -; LARGE-NEXT: ldx.d $ra, $ra, $a1 +; LARGE-NEXT: pcalau12i $ra, %got_pc_hi20(callee) +; LARGE-NEXT: addi.d $t8, $zero, %got_pc_lo12(callee) +; LARGE-NEXT: lu32i.d $t8, %got64_pc_lo20(callee) +; LARGE-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(callee) +; LARGE-NEXT: ldx.d $ra, $t8, $ra ; LARGE-NEXT: jirl $ra, $ra, 0 ; LARGE-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LARGE-NEXT: addi.d $sp, $sp, 16 @@ -82,11 +82,11 @@ define void @call_external_sym(ptr %dst) { ; LARGE-NEXT: .cfi_offset 1, -8 ; LARGE-NEXT: ori $a2, $zero, 1000 ; LARGE-NEXT: move $a1, $zero -; LARGE-NEXT: pcalau12i $a3, %pc_hi20(memset) -; LARGE-NEXT: addi.d $ra, $zero, %pc_lo12(memset) -; LARGE-NEXT: lu32i.d $ra, %pc64_lo20(memset) -; LARGE-NEXT: lu52i.d $ra, $ra, %pc64_hi12(memset) -; LARGE-NEXT: add.d $ra, $ra, $a3 +; LARGE-NEXT: pcalau12i $ra, %pc_hi20(memset) +; LARGE-NEXT: addi.d $t8, $zero, %pc_lo12(memset) +; LARGE-NEXT: lu32i.d $t8, %pc64_lo20(memset) +; LARGE-NEXT: lu52i.d $t8, $t8, %pc64_hi12(memset) +; LARGE-NEXT: add.d $ra, $t8, $ra ; LARGE-NEXT: jirl $ra, $ra, 0 ; LARGE-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LARGE-NEXT: addi.d $sp, $sp, 16 @@ -105,17 +105,17 @@ define i32 @caller_tail(i32 %i) nounwind { ; ; MEDIUM-LABEL: caller_tail: ; MEDIUM: # %bb.0: # %entry -; MEDIUM-NEXT: pcaddu18i $a1, %call36(callee_tail) -; MEDIUM-NEXT: jr $a1 +; MEDIUM-NEXT: pcaddu18i $t8, %call36(callee_tail) +; MEDIUM-NEXT: jr $t8 ; ; LARGE-LABEL: caller_tail: ; LARGE: # %bb.0: # %entry -; LARGE-NEXT: pcalau12i $a1, %got_pc_hi20(callee_tail) -; LARGE-NEXT: addi.d $a2, $zero, %got_pc_lo12(callee_tail) -; LARGE-NEXT: lu32i.d $a2, %got64_pc_lo20(callee_tail) -; LARGE-NEXT: lu52i.d $a2, $a2, %got64_pc_hi12(callee_tail) -; LARGE-NEXT: ldx.d $a1, $a2, $a1 -; LARGE-NEXT: jr $a1 +; LARGE-NEXT: pcalau12i $t7, %got_pc_hi20(callee_tail) +; LARGE-NEXT: addi.d $t8, $zero, %got_pc_lo12(callee_tail) +; LARGE-NEXT: lu32i.d $t8, %got64_pc_lo20(callee_tail) +; LARGE-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(callee_tail) +; LARGE-NEXT: ldx.d $t7, $t8, $t7 +; LARGE-NEXT: jr $t7 entry: %r = tail call i32 @callee_tail(i32 %i) ret i32 %r diff --git a/llvm/test/CodeGen/LoongArch/expand-call.ll b/llvm/test/CodeGen/LoongArch/expand-call.ll index 86bf4292665b..e0d179f92de6 100644 --- a/llvm/test/CodeGen/LoongArch/expand-call.ll +++ b/llvm/test/CodeGen/LoongArch/expand-call.ll @@ -1,6 +1,6 @@ ; RUN: llc --mtriple=loongarch64 --stop-before loongarch-prera-expand-pseudo \ ; RUN: --verify-machineinstrs < %s | FileCheck %s --check-prefix=NOEXPAND -; RUN: llc --mtriple=loongarch64 --stop-after loongarch-prera-expand-pseudo \ +; RUN: llc --mtriple=loongarch64 --stop-before machine-opt-remark-emitter \ ; RUN: --verify-machineinstrs < %s | FileCheck %s --check-prefix=EXPAND declare void @callee() diff --git a/llvm/test/CodeGen/LoongArch/global-address.ll b/llvm/test/CodeGen/LoongArch/global-address.ll index a8f0ef648aa7..d32a17f488b1 100644 --- a/llvm/test/CodeGen/LoongArch/global-address.ll +++ b/llvm/test/CodeGen/LoongArch/global-address.ll @@ -53,32 +53,32 @@ define void @foo() nounwind { ; LA64LARGENOPIC-LABEL: foo: ; LA64LARGENOPIC: # %bb.0: ; LA64LARGENOPIC-NEXT: pcalau12i $a0, %got_pc_hi20(G) -; LA64LARGENOPIC-NEXT: addi.d $a1, $zero, %got_pc_lo12(G) -; LA64LARGENOPIC-NEXT: lu32i.d $a1, %got64_pc_lo20(G) -; LA64LARGENOPIC-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(G) -; LA64LARGENOPIC-NEXT: ldx.d $a0, $a1, $a0 +; LA64LARGENOPIC-NEXT: addi.d $t8, $zero, %got_pc_lo12(G) +; LA64LARGENOPIC-NEXT: lu32i.d $t8, %got64_pc_lo20(G) +; LA64LARGENOPIC-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) +; LA64LARGENOPIC-NEXT: ldx.d $a0, $t8, $a0 ; LA64LARGENOPIC-NEXT: ld.w $a0, $a0, 0 ; LA64LARGENOPIC-NEXT: pcalau12i $a0, %pc_hi20(g) -; LA64LARGENOPIC-NEXT: addi.d $a1, $zero, %pc_lo12(g) -; LA64LARGENOPIC-NEXT: lu32i.d $a1, %pc64_lo20(g) -; LA64LARGENOPIC-NEXT: lu52i.d $a1, $a1, %pc64_hi12(g) -; LA64LARGENOPIC-NEXT: add.d $a0, $a1, $a0 +; LA64LARGENOPIC-NEXT: addi.d $t8, $zero, %pc_lo12(g) +; LA64LARGENOPIC-NEXT: lu32i.d $t8, %pc64_lo20(g) +; LA64LARGENOPIC-NEXT: lu52i.d $t8, $t8, %pc64_hi12(g) +; LA64LARGENOPIC-NEXT: add.d $a0, $t8, $a0 ; LA64LARGENOPIC-NEXT: ld.w $a0, $a0, 0 ; LA64LARGENOPIC-NEXT: ret ; ; LA64LARGEPIC-LABEL: foo: ; LA64LARGEPIC: # %bb.0: ; LA64LARGEPIC-NEXT: pcalau12i $a0, %got_pc_hi20(G) -; LA64LARGEPIC-NEXT: addi.d $a1, $zero, %got_pc_lo12(G) -; LA64LARGEPIC-NEXT: lu32i.d $a1, %got64_pc_lo20(G) -; LA64LARGEPIC-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(G) -; LA64LARGEPIC-NEXT: ldx.d $a0, $a1, $a0 +; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %got_pc_lo12(G) +; LA64LARGEPIC-NEXT: lu32i.d $t8, %got64_pc_lo20(G) +; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) +; LA64LARGEPIC-NEXT: ldx.d $a0, $t8, $a0 ; LA64LARGEPIC-NEXT: ld.w $a0, $a0, 0 ; LA64LARGEPIC-NEXT: pcalau12i $a0, %pc_hi20(.Lg$local) -; LA64LARGEPIC-NEXT: addi.d $a1, $zero, %pc_lo12(.Lg$local) -; LA64LARGEPIC-NEXT: lu32i.d $a1, %pc64_lo20(.Lg$local) -; LA64LARGEPIC-NEXT: lu52i.d $a1, $a1, %pc64_hi12(.Lg$local) -; LA64LARGEPIC-NEXT: add.d $a0, $a1, $a0 +; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %pc_lo12(.Lg$local) +; LA64LARGEPIC-NEXT: lu32i.d $t8, %pc64_lo20(.Lg$local) +; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %pc64_hi12(.Lg$local) +; LA64LARGEPIC-NEXT: add.d $a0, $t8, $a0 ; LA64LARGEPIC-NEXT: ld.w $a0, $a0, 0 ; LA64LARGEPIC-NEXT: ret %V = load volatile i32, ptr @G diff --git a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll index 150a935d7bf8..d6f3e3469f75 100644 --- a/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll +++ b/llvm/test/CodeGen/LoongArch/psabi-restricted-scheduling.ll @@ -52,13 +52,13 @@ define void @foo() nounwind { ; MEDIUM_SCH-NEXT: addi.d $sp, $sp, -16 ; MEDIUM_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill ; MEDIUM_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) -; MEDIUM_SCH-NEXT: pcaddu18i $ra, %call36(bar) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %got_pc_lo12(G) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, 0 ; MEDIUM_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) ; MEDIUM_SCH-NEXT: addi.d $a0, $a0, %pc_lo12(g) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, 0 ; MEDIUM_SCH-NEXT: ori $a0, $zero, 1 +; MEDIUM_SCH-NEXT: pcaddu18i $ra, %call36(bar) ; MEDIUM_SCH-NEXT: jirl $ra, $ra, 0 ; MEDIUM_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) ; MEDIUM_SCH-NEXT: ld.d $a0, $a0, %ie_pc_lo12(gd) @@ -78,41 +78,41 @@ define void @foo() nounwind { ; LARGE_NO_SCH-NEXT: addi.d $sp, $sp, -16 ; LARGE_NO_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) -; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %got_pc_lo12(G) -; LARGE_NO_SCH-NEXT: lu32i.d $a1, %got64_pc_lo20(G) -; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(G) -; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %got_pc_lo12(G) +; LARGE_NO_SCH-NEXT: lu32i.d $t8, %got64_pc_lo20(G) +; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_NO_SCH-NEXT: ld.d $a0, $a0, 0 ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) -; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %pc_lo12(g) -; LARGE_NO_SCH-NEXT: lu32i.d $a1, %pc64_lo20(g) -; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %pc64_hi12(g) -; LARGE_NO_SCH-NEXT: add.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %pc_lo12(g) +; LARGE_NO_SCH-NEXT: lu32i.d $t8, %pc64_lo20(g) +; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %pc64_hi12(g) +; LARGE_NO_SCH-NEXT: add.d $a0, $t8, $a0 ; LARGE_NO_SCH-NEXT: ld.d $a0, $a0, 0 ; LARGE_NO_SCH-NEXT: ori $a0, $zero, 1 -; LARGE_NO_SCH-NEXT: pcalau12i $a1, %got_pc_hi20(bar) -; LARGE_NO_SCH-NEXT: addi.d $ra, $zero, %got_pc_lo12(bar) -; LARGE_NO_SCH-NEXT: lu32i.d $ra, %got64_pc_lo20(bar) -; LARGE_NO_SCH-NEXT: lu52i.d $ra, $ra, %got64_pc_hi12(bar) -; LARGE_NO_SCH-NEXT: ldx.d $ra, $ra, $a1 +; LARGE_NO_SCH-NEXT: pcalau12i $ra, %got_pc_hi20(bar) +; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %got_pc_lo12(bar) +; LARGE_NO_SCH-NEXT: lu32i.d $t8, %got64_pc_lo20(bar) +; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(bar) +; LARGE_NO_SCH-NEXT: ldx.d $ra, $t8, $ra ; LARGE_NO_SCH-NEXT: jirl $ra, $ra, 0 ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) -; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(gd) -; LARGE_NO_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(gd) -; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(gd) -; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(gd) +; LARGE_NO_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(gd) +; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(gd) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) -; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ld) -; LARGE_NO_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ld) -; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ld) -; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ld) +; LARGE_NO_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ld) +; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ld) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp ; LARGE_NO_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) -; LARGE_NO_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ie) -; LARGE_NO_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ie) -; LARGE_NO_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ie) -; LARGE_NO_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_NO_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ie) +; LARGE_NO_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) +; LARGE_NO_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) +; LARGE_NO_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_NO_SCH-NEXT: ldx.d $a0, $a0, $tp ; LARGE_NO_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LARGE_NO_SCH-NEXT: addi.d $sp, $sp, 16 @@ -122,42 +122,42 @@ define void @foo() nounwind { ; LARGE_SCH: # %bb.0: ; LARGE_SCH-NEXT: addi.d $sp, $sp, -16 ; LARGE_SCH-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill -; LARGE_SCH-NEXT: addi.d $a1, $zero, %got_pc_lo12(G) ; LARGE_SCH-NEXT: pcalau12i $a0, %got_pc_hi20(G) -; LARGE_SCH-NEXT: addi.d $ra, $zero, %got_pc_lo12(bar) -; LARGE_SCH-NEXT: lu32i.d $a1, %got64_pc_lo20(G) -; LARGE_SCH-NEXT: lu32i.d $ra, %got64_pc_lo20(bar) -; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(G) -; LARGE_SCH-NEXT: lu52i.d $ra, $ra, %got64_pc_hi12(bar) -; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 -; LARGE_SCH-NEXT: addi.d $a1, $zero, %pc_lo12(g) -; LARGE_SCH-NEXT: lu32i.d $a1, %pc64_lo20(g) -; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %pc64_hi12(g) +; LARGE_SCH-NEXT: addi.d $t8, $zero, %got_pc_lo12(G) +; LARGE_SCH-NEXT: lu32i.d $t8, %got64_pc_lo20(G) +; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(G) +; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_SCH-NEXT: ld.d $a0, $a0, 0 ; LARGE_SCH-NEXT: pcalau12i $a0, %pc_hi20(g) -; LARGE_SCH-NEXT: add.d $a0, $a1, $a0 -; LARGE_SCH-NEXT: pcalau12i $a1, %got_pc_hi20(bar) +; LARGE_SCH-NEXT: addi.d $t8, $zero, %pc_lo12(g) +; LARGE_SCH-NEXT: lu32i.d $t8, %pc64_lo20(g) +; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %pc64_hi12(g) +; LARGE_SCH-NEXT: add.d $a0, $t8, $a0 ; LARGE_SCH-NEXT: ld.d $a0, $a0, 0 -; LARGE_SCH-NEXT: ldx.d $ra, $ra, $a1 ; LARGE_SCH-NEXT: ori $a0, $zero, 1 +; LARGE_SCH-NEXT: pcalau12i $ra, %got_pc_hi20(bar) +; LARGE_SCH-NEXT: addi.d $t8, $zero, %got_pc_lo12(bar) +; LARGE_SCH-NEXT: lu32i.d $t8, %got64_pc_lo20(bar) +; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(bar) +; LARGE_SCH-NEXT: ldx.d $ra, $t8, $ra ; LARGE_SCH-NEXT: jirl $ra, $ra, 0 -; LARGE_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(gd) ; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(gd) -; LARGE_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(gd) -; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(gd) -; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 -; LARGE_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ld) -; LARGE_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ld) -; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ld) +; LARGE_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(gd) +; LARGE_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(gd) +; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(gd) +; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp ; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) -; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 -; LARGE_SCH-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ie) -; LARGE_SCH-NEXT: lu32i.d $a1, %ie64_pc_lo20(ie) -; LARGE_SCH-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ie) +; LARGE_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ld) +; LARGE_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ld) +; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ld) +; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp ; LARGE_SCH-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) -; LARGE_SCH-NEXT: ldx.d $a0, $a1, $a0 +; LARGE_SCH-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ie) +; LARGE_SCH-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) +; LARGE_SCH-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) +; LARGE_SCH-NEXT: ldx.d $a0, $t8, $a0 ; LARGE_SCH-NEXT: ldx.d $a0, $a0, $tp ; LARGE_SCH-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LARGE_SCH-NEXT: addi.d $sp, $sp, 16 diff --git a/llvm/test/CodeGen/LoongArch/tls-models.ll b/llvm/test/CodeGen/LoongArch/tls-models.ll index a2a3792a6a54..3994df1da716 100644 --- a/llvm/test/CodeGen/LoongArch/tls-models.ll +++ b/llvm/test/CodeGen/LoongArch/tls-models.ll @@ -45,15 +45,15 @@ define ptr @f1() nounwind { ; LA64LARGEPIC-NEXT: addi.d $sp, $sp, -16 ; LA64LARGEPIC-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill ; LA64LARGEPIC-NEXT: pcalau12i $a0, %gd_pc_hi20(unspecified) -; LA64LARGEPIC-NEXT: addi.d $a1, $zero, %got_pc_lo12(unspecified) -; LA64LARGEPIC-NEXT: lu32i.d $a1, %got64_pc_lo20(unspecified) -; LA64LARGEPIC-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(unspecified) -; LA64LARGEPIC-NEXT: add.d $a0, $a1, $a0 -; LA64LARGEPIC-NEXT: pcalau12i $a1, %pc_hi20(__tls_get_addr) -; LA64LARGEPIC-NEXT: addi.d $ra, $zero, %pc_lo12(__tls_get_addr) -; LA64LARGEPIC-NEXT: lu32i.d $ra, %pc64_lo20(__tls_get_addr) -; LA64LARGEPIC-NEXT: lu52i.d $ra, $ra, %pc64_hi12(__tls_get_addr) -; LA64LARGEPIC-NEXT: add.d $ra, $ra, $a1 +; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %got_pc_lo12(unspecified) +; LA64LARGEPIC-NEXT: lu32i.d $t8, %got64_pc_lo20(unspecified) +; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(unspecified) +; LA64LARGEPIC-NEXT: add.d $a0, $t8, $a0 +; LA64LARGEPIC-NEXT: pcalau12i $ra, %pc_hi20(__tls_get_addr) +; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %pc_lo12(__tls_get_addr) +; LA64LARGEPIC-NEXT: lu32i.d $t8, %pc64_lo20(__tls_get_addr) +; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr) +; LA64LARGEPIC-NEXT: add.d $ra, $t8, $ra ; LA64LARGEPIC-NEXT: jirl $ra, $ra, 0 ; LA64LARGEPIC-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LA64LARGEPIC-NEXT: addi.d $sp, $sp, 16 @@ -76,10 +76,10 @@ define ptr @f1() nounwind { ; LA64LARGENOPIC-LABEL: f1: ; LA64LARGENOPIC: # %bb.0: # %entry ; LA64LARGENOPIC-NEXT: pcalau12i $a0, %ie_pc_hi20(unspecified) -; LA64LARGENOPIC-NEXT: addi.d $a1, $zero, %ie_pc_lo12(unspecified) -; LA64LARGENOPIC-NEXT: lu32i.d $a1, %ie64_pc_lo20(unspecified) -; LA64LARGENOPIC-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(unspecified) -; LA64LARGENOPIC-NEXT: ldx.d $a0, $a1, $a0 +; LA64LARGENOPIC-NEXT: addi.d $t8, $zero, %ie_pc_lo12(unspecified) +; LA64LARGENOPIC-NEXT: lu32i.d $t8, %ie64_pc_lo20(unspecified) +; LA64LARGENOPIC-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(unspecified) +; LA64LARGENOPIC-NEXT: ldx.d $a0, $t8, $a0 ; LA64LARGENOPIC-NEXT: add.d $a0, $a0, $tp ; LA64LARGENOPIC-NEXT: ret entry: @@ -116,15 +116,15 @@ define ptr @f2() nounwind { ; LA64LARGEPIC-NEXT: addi.d $sp, $sp, -16 ; LA64LARGEPIC-NEXT: st.d $ra, $sp, 8 # 8-byte Folded Spill ; LA64LARGEPIC-NEXT: pcalau12i $a0, %ld_pc_hi20(ld) -; LA64LARGEPIC-NEXT: addi.d $a1, $zero, %got_pc_lo12(ld) -; LA64LARGEPIC-NEXT: lu32i.d $a1, %got64_pc_lo20(ld) -; LA64LARGEPIC-NEXT: lu52i.d $a1, $a1, %got64_pc_hi12(ld) -; LA64LARGEPIC-NEXT: add.d $a0, $a1, $a0 -; LA64LARGEPIC-NEXT: pcalau12i $a1, %pc_hi20(__tls_get_addr) -; LA64LARGEPIC-NEXT: addi.d $ra, $zero, %pc_lo12(__tls_get_addr) -; LA64LARGEPIC-NEXT: lu32i.d $ra, %pc64_lo20(__tls_get_addr) -; LA64LARGEPIC-NEXT: lu52i.d $ra, $ra, %pc64_hi12(__tls_get_addr) -; LA64LARGEPIC-NEXT: add.d $ra, $ra, $a1 +; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %got_pc_lo12(ld) +; LA64LARGEPIC-NEXT: lu32i.d $t8, %got64_pc_lo20(ld) +; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %got64_pc_hi12(ld) +; LA64LARGEPIC-NEXT: add.d $a0, $t8, $a0 +; LA64LARGEPIC-NEXT: pcalau12i $ra, %pc_hi20(__tls_get_addr) +; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %pc_lo12(__tls_get_addr) +; LA64LARGEPIC-NEXT: lu32i.d $t8, %pc64_lo20(__tls_get_addr) +; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %pc64_hi12(__tls_get_addr) +; LA64LARGEPIC-NEXT: add.d $ra, $t8, $ra ; LA64LARGEPIC-NEXT: jirl $ra, $ra, 0 ; LA64LARGEPIC-NEXT: ld.d $ra, $sp, 8 # 8-byte Folded Reload ; LA64LARGEPIC-NEXT: addi.d $sp, $sp, 16 @@ -147,10 +147,10 @@ define ptr @f2() nounwind { ; LA64LARGENOPIC-LABEL: f2: ; LA64LARGENOPIC: # %bb.0: # %entry ; LA64LARGENOPIC-NEXT: pcalau12i $a0, %ie_pc_hi20(ld) -; LA64LARGENOPIC-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ld) -; LA64LARGENOPIC-NEXT: lu32i.d $a1, %ie64_pc_lo20(ld) -; LA64LARGENOPIC-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ld) -; LA64LARGENOPIC-NEXT: ldx.d $a0, $a1, $a0 +; LA64LARGENOPIC-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ld) +; LA64LARGENOPIC-NEXT: lu32i.d $t8, %ie64_pc_lo20(ld) +; LA64LARGENOPIC-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ld) +; LA64LARGENOPIC-NEXT: ldx.d $a0, $t8, $a0 ; LA64LARGENOPIC-NEXT: add.d $a0, $a0, $tp ; LA64LARGENOPIC-NEXT: ret entry: @@ -177,10 +177,10 @@ define ptr @f3() nounwind { ; LA64LARGEPIC-LABEL: f3: ; LA64LARGEPIC: # %bb.0: # %entry ; LA64LARGEPIC-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) -; LA64LARGEPIC-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ie) -; LA64LARGEPIC-NEXT: lu32i.d $a1, %ie64_pc_lo20(ie) -; LA64LARGEPIC-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ie) -; LA64LARGEPIC-NEXT: ldx.d $a0, $a1, $a0 +; LA64LARGEPIC-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ie) +; LA64LARGEPIC-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) +; LA64LARGEPIC-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) +; LA64LARGEPIC-NEXT: ldx.d $a0, $t8, $a0 ; LA64LARGEPIC-NEXT: add.d $a0, $a0, $tp ; LA64LARGEPIC-NEXT: ret ; @@ -201,10 +201,10 @@ define ptr @f3() nounwind { ; LA64LARGENOPIC-LABEL: f3: ; LA64LARGENOPIC: # %bb.0: # %entry ; LA64LARGENOPIC-NEXT: pcalau12i $a0, %ie_pc_hi20(ie) -; LA64LARGENOPIC-NEXT: addi.d $a1, $zero, %ie_pc_lo12(ie) -; LA64LARGENOPIC-NEXT: lu32i.d $a1, %ie64_pc_lo20(ie) -; LA64LARGENOPIC-NEXT: lu52i.d $a1, $a1, %ie64_pc_hi12(ie) -; LA64LARGENOPIC-NEXT: ldx.d $a0, $a1, $a0 +; LA64LARGENOPIC-NEXT: addi.d $t8, $zero, %ie_pc_lo12(ie) +; LA64LARGENOPIC-NEXT: lu32i.d $t8, %ie64_pc_lo20(ie) +; LA64LARGENOPIC-NEXT: lu52i.d $t8, $t8, %ie64_pc_hi12(ie) +; LA64LARGENOPIC-NEXT: ldx.d $a0, $t8, $a0 ; LA64LARGENOPIC-NEXT: add.d $a0, $a0, $tp ; LA64LARGENOPIC-NEXT: ret entry: -- GitLab From f68647997b828eeaa580e74e1d7c8565a9ecd215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Ferenc=20Szab=C3=B3?= <30732159+dfszabo@users.noreply.github.com> Date: Fri, 5 Jan 2024 04:13:39 +0100 Subject: [PATCH 064/728] =?UTF-8?q?[GlobalISel]=20Adding=20support=20for?= =?UTF-8?q?=20handling=20G=5FASSERT=5F{SEXT,ZEXT,ALIGN}=20i=E2=80=A6=20(#7?= =?UTF-8?q?4196)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …n artifact combiner These instructions are hint for optimizations and can be treated as copies and are handled as such with this change. Without it is possible to run into an assertion, since tryCombineUnmergeValues rightfully use getDefIgnoringCopies to get the source MI, which already handle these hint instructions and treat them as copies. The problem is that markDefDead only considers COPYs, which will lead to crash with assertion for cases like testUnmergeHintOfTrunc. --- .../GlobalISel/LegalizationArtifactCombiner.h | 21 +++--- .../irtranslator-hoisted-constants.ll | 5 +- .../AArch64/GlobalISel/legalize-simple.mir | 64 +++++++++++++++++++ 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 851353042cc2..da330b517c28 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -1366,6 +1366,9 @@ public: // Adding Use to ArtifactList. WrapperObserver.changedInstr(Use); break; + case TargetOpcode::G_ASSERT_SEXT: + case TargetOpcode::G_ASSERT_ZEXT: + case TargetOpcode::G_ASSERT_ALIGN: case TargetOpcode::COPY: { Register Copy = Use.getOperand(0).getReg(); if (Copy.isVirtual()) @@ -1392,6 +1395,9 @@ private: case TargetOpcode::G_ANYEXT: case TargetOpcode::G_SEXT: case TargetOpcode::G_EXTRACT: + case TargetOpcode::G_ASSERT_SEXT: + case TargetOpcode::G_ASSERT_ZEXT: + case TargetOpcode::G_ASSERT_ALIGN: return MI.getOperand(1).getReg(); case TargetOpcode::G_UNMERGE_VALUES: return MI.getOperand(MI.getNumOperands() - 1).getReg(); @@ -1425,7 +1431,8 @@ private: if (MRI.hasOneUse(PrevRegSrc)) { if (TmpDef != &DefMI) { assert((TmpDef->getOpcode() == TargetOpcode::COPY || - isArtifactCast(TmpDef->getOpcode())) && + isArtifactCast(TmpDef->getOpcode()) || + isPreISelGenericOptimizationHint(TmpDef->getOpcode())) && "Expecting copy or artifact cast here"); DeadInsts.push_back(TmpDef); @@ -1509,16 +1516,8 @@ private: /// Looks through copy instructions and returns the actual /// source register. Register lookThroughCopyInstrs(Register Reg) { - using namespace llvm::MIPatternMatch; - - Register TmpReg; - while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) { - if (MRI.getType(TmpReg).isValid()) - Reg = TmpReg; - else - break; - } - return Reg; + Register TmpReg = getSrcRegIgnoringCopies(Reg, MRI); + return TmpReg.isValid() ? TmpReg : Reg; } }; diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-hoisted-constants.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-hoisted-constants.ll index a7d7a1e81617..5867326c18aa 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-hoisted-constants.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-hoisted-constants.ll @@ -46,14 +46,11 @@ define i32 @test(i32 %a, i1 %c) { ; PRESELECTION-NEXT: {{ $}} ; PRESELECTION-NEXT: [[COPY:%[0-9]+]]:gpr(s32) = COPY $w0 ; PRESELECTION-NEXT: [[COPY1:%[0-9]+]]:gpr(s32) = COPY $w1 - ; PRESELECTION-NEXT: [[TRUNC:%[0-9]+]]:gpr(s8) = G_TRUNC [[COPY1]](s32) - ; PRESELECTION-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:gpr(s8) = G_ASSERT_ZEXT [[TRUNC]], 1 ; PRESELECTION-NEXT: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 0 ; PRESELECTION-NEXT: [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 100000 ; PRESELECTION-NEXT: [[CONSTANT_FOLD_BARRIER:%[0-9]+]]:gpr(s32) = G_CONSTANT_FOLD_BARRIER [[C1]] - ; PRESELECTION-NEXT: [[ANYEXT:%[0-9]+]]:gpr(s32) = G_ANYEXT [[ASSERT_ZEXT]](s8) ; PRESELECTION-NEXT: [[C2:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1 - ; PRESELECTION-NEXT: [[AND:%[0-9]+]]:gpr(s32) = G_AND [[ANYEXT]], [[C2]] + ; PRESELECTION-NEXT: [[AND:%[0-9]+]]:gpr(s32) = G_AND [[COPY1]], [[C2]] ; PRESELECTION-NEXT: G_BRCOND [[AND]](s32), %bb.3 ; PRESELECTION-NEXT: G_BR %bb.2 ; PRESELECTION-NEXT: {{ $}} diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir index 62865bcfdf08..ce02aa380efe 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir @@ -136,6 +136,47 @@ body: | $x0 = COPY %3(s64) RET_ReallyLR implicit $x0 +... +--- +name: testExtOfHintOfTrunc +body: | + bb.0: + liveins: $x0 + + ; CHECK-LABEL: name: testExtOfHintOfTrunc + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY [[COPY]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:_(s64) = COPY $x0 + %1:_(s8) = G_TRUNC %0(s64) + %2:_(s8) = G_ASSERT_ZEXT %1(s8), 1 + %3:_(s64) = G_ANYEXT %2(s8) + $x0 = COPY %3(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: testExtOfCopyAndHintOfTrunc +body: | + bb.0: + liveins: $x0 + + ; CHECK-LABEL: name: testExtOfCopyAndHintOfTrunc + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY [[COPY]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:_(s64) = COPY $x0 + %1:_(s8) = G_TRUNC %0(s64) + %2:_(s8) = COPY %1(s8) + %3:_(s8) = G_ASSERT_ZEXT %2(s8), 1 + %4:_(s64) = G_ANYEXT %3(s8) + $x0 = COPY %4(s64) + RET_ReallyLR implicit $x0 + ... --- name: testExtOf2CopyOfTrunc @@ -158,3 +199,26 @@ body: | RET_ReallyLR implicit $x0 ... +--- +name: testUnmergeHintOfTrunc +body: | + bb.0: + liveins: $x0 + + ; CHECK-LABEL: name: testUnmergeHintOfTrunc + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8), [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[COPY]](s64) + ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UV1]](s8) + ; CHECK-NEXT: $x0 = COPY [[ANYEXT]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:_(s64) = COPY $x0 + %1:_(s16) = G_TRUNC %0(s64) + %2:_(s16) = G_ASSERT_ZEXT %1(s16), 1 + %3:_(s8), %4:_(s8) = G_UNMERGE_VALUES %2(s16) + %5:_(s64) = G_ANYEXT %4(s8) + $x0 = COPY %5(s64) + RET_ReallyLR implicit $x0 + +... -- GitLab From b4cfb50c65bd3893d1a0639deb33bd664214a20f Mon Sep 17 00:00:00 2001 From: Michal Paszkowski Date: Thu, 4 Jan 2024 19:31:15 -0800 Subject: [PATCH 065/728] [SPIR-V] Emit SPIR-V bitcasts between source/expected pointer type (#69621) This patch introduces a new spv_ptrcast intrinsic for tracking expected pointer types. The change fixes multiple OpenCL CTS regressions due the switch to opaque pointers (e.g. basic/hiloeo). --- llvm/include/llvm/IR/IntrinsicsSPIRV.td | 1 + llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 7 +- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 125 +++++++++++++++++- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp | 24 +++- .../CodeGen/SPIRV/AtomicCompareExchange.ll | 31 ++--- .../SPIRV/function/alloca-load-store.ll | 22 +-- .../undef-nested-composite-store.ll | 3 +- .../undef-simple-composite-store.ll | 3 +- .../CodeGen/SPIRV/llvm-intrinsics/memset.ll | 4 +- llvm/test/CodeGen/SPIRV/opaque_pointers.ll | 20 +-- .../SPIRV/opencl/basic/get_global_offset.ll | 8 +- ...er-type-deduction-no-bitcast-to-generic.ll | 19 +++ .../SPIRV/pointers/two-bitcast-users.ll | 19 +++ .../SPIRV/pointers/two-subsequent-bitcasts.ll | 17 +++ llvm/test/CodeGen/SPIRV/sitofp-with-bool.ll | 15 ++- .../OpPhi_ArgumentsPlaceholders.ll | 1 + .../CodeGen/SPIRV/transcoding/spec_const.ll | 1 + .../spirv-private-array-initialization.ll | 6 +- llvm/test/CodeGen/SPIRV/uitofp-with-bool.ll | 4 +- 19 files changed, 264 insertions(+), 66 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll create mode 100644 llvm/test/CodeGen/SPIRV/pointers/two-bitcast-users.ll create mode 100644 llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td index 736be8ca3212..ea0074d22a44 100644 --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -28,6 +28,7 @@ let TargetPrefix = "spv" in { def int_spv_insertelt : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_any_ty, llvm_anyint_ty]>; def int_spv_const_composite : Intrinsic<[llvm_i32_ty], [llvm_vararg_ty]>; def int_spv_bitcast : Intrinsic<[llvm_any_ty], [llvm_any_ty]>; + def int_spv_ptrcast : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg>]>; def int_spv_switch : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>; def int_spv_cmpxchg : Intrinsic<[llvm_i32_ty], [llvm_any_ty, llvm_vararg_ty]>; def int_spv_unreachable : Intrinsic<[], []>; diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp index 629db8e2eb4d..0a8b5499a1fc 100644 --- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp @@ -211,8 +211,8 @@ static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx, MDString *MDKernelArgType = getKernelArgAttribute(F, ArgIdx, "kernel_arg_type"); - if (!MDKernelArgType || (MDKernelArgType->getString().ends_with("*") && - MDKernelArgType->getString().ends_with("_t"))) + if (!MDKernelArgType || (!MDKernelArgType->getString().ends_with("*") && + !MDKernelArgType->getString().ends_with("_t"))) return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual); if (MDKernelArgType->getString().ends_with("*")) @@ -438,7 +438,8 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs"); ArgVRegs.push_back(Arg.Regs[0]); SPIRVType *SPIRVTy = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder); - GR->assignSPIRVTypeToVReg(SPIRVTy, Arg.Regs[0], MIRBuilder.getMF()); + if (!GR->getSPIRVTypeForVReg(Arg.Regs[0])) + GR->assignSPIRVTypeToVReg(SPIRVTy, Arg.Regs[0], MIRBuilder.getMF()); } if (auto Res = SPIRV::lowerBuiltin( DemangledName, SPIRV::InstructionSet::OpenCL_std, MIRBuilder, diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 660c574daf38..fb4e9932dd2d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -74,6 +74,7 @@ class SPIRVEmitIntrinsics void processInstrAfterVisit(Instruction *I); void insertAssignPtrTypeIntrs(Instruction *I); void insertAssignTypeIntrs(Instruction *I); + void insertPtrCastInstr(Instruction *I); void processGlobalValue(GlobalVariable &GV); public: @@ -255,7 +256,19 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) { } Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { - SmallVector Types = {I.getType(), I.getOperand(0)->getType()}; + Value *Source = I.getOperand(0); + + // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of + // varying element types. In case of IR coming from older versions of LLVM + // such bitcasts do not provide sufficient information, should be just skipped + // here, and handled in insertPtrCastInstr. + if (I.getType()->isPointerTy()) { + I.replaceAllUsesWith(Source); + I.eraseFromParent(); + return nullptr; + } + + SmallVector Types = {I.getType(), Source->getType()}; SmallVector Args(I.op_begin(), I.op_end()); auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args}); std::string InstName = I.hasName() ? I.getName().str() : ""; @@ -265,6 +278,111 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { return NewI; } +void SPIRVEmitIntrinsics::insertPtrCastInstr(Instruction *I) { + Value *Pointer; + Type *ExpectedElementType; + unsigned OperandToReplace; + if (StoreInst *SI = dyn_cast(I)) { + Pointer = SI->getPointerOperand(); + ExpectedElementType = SI->getValueOperand()->getType(); + OperandToReplace = 1; + } else if (LoadInst *LI = dyn_cast(I)) { + Pointer = LI->getPointerOperand(); + ExpectedElementType = LI->getType(); + OperandToReplace = 0; + } else if (GetElementPtrInst *GEPI = dyn_cast(I)) { + Pointer = GEPI->getPointerOperand(); + ExpectedElementType = GEPI->getSourceElementType(); + OperandToReplace = 0; + } else { + return; + } + + // If Pointer is the result of nop BitCastInst (ptr -> ptr), use the source + // pointer instead. The BitCastInst should be later removed when visited. + while (BitCastInst *BC = dyn_cast(Pointer)) + Pointer = BC->getOperand(0); + + // Do not emit spv_ptrcast if Pointer is a GlobalValue of expected type. + GlobalValue *GV = dyn_cast(Pointer); + if (GV && GV->getValueType() == ExpectedElementType) + return; + + // Do not emit spv_ptrcast if Pointer is a result of alloca with expected + // type. + AllocaInst *A = dyn_cast(Pointer); + if (A && A->getAllocatedType() == ExpectedElementType) + return; + + if (dyn_cast(Pointer)) + return; + + setInsertPointSkippingPhis(*IRB, I); + Constant *ExpectedElementTypeConst = + Constant::getNullValue(ExpectedElementType); + ConstantAsMetadata *CM = + ValueAsMetadata::getConstant(ExpectedElementTypeConst); + MDTuple *TyMD = MDNode::get(F->getContext(), CM); + MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD); + unsigned AddressSpace = Pointer->getType()->getPointerAddressSpace(); + bool FirstPtrCastOrAssignPtrType = true; + + // Do not emit new spv_ptrcast if equivalent one already exists or when + // spv_assign_ptr_type already targets this pointer with the same element + // type. + for (auto User : Pointer->users()) { + auto *II = dyn_cast(User); + if (!II || + (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type && + II->getIntrinsicID() != Intrinsic::spv_ptrcast) || + II->getOperand(0) != Pointer) + continue; + + // There is some spv_ptrcast/spv_assign_ptr_type already targeting this + // pointer. + FirstPtrCastOrAssignPtrType = false; + if (II->getOperand(1) != VMD || + dyn_cast(II->getOperand(2))->getSExtValue() != + AddressSpace) + continue; + + // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the same + // element type and address space. + if (II->getIntrinsicID() != Intrinsic::spv_ptrcast) + return; + + // This must be a spv_ptrcast, do not emit new if this one has the same BB + // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type. + if (II->getParent() != I->getParent()) + continue; + + I->setOperand(OperandToReplace, II); + return; + } + + // Do not emit spv_ptrcast if it would cast to the default pointer element + // type (i8) of the same address space. + if (ExpectedElementType->isIntegerTy(8)) + return; + + // If this would be the first spv_ptrcast and there is no spv_assign_ptr_type + // for this pointer before, do not emit spv_ptrcast but emit + // spv_assign_ptr_type instead. + if (FirstPtrCastOrAssignPtrType && isa(Pointer)) { + buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {Pointer->getType()}, + ExpectedElementTypeConst, Pointer, + {IRB->getInt32(AddressSpace)}); + return; + } else { + SmallVector Types = {Pointer->getType(), Pointer->getType()}; + SmallVector Args = {Pointer, VMD, IRB->getInt32(AddressSpace)}; + auto *PtrCastI = + IRB->CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); + I->setOperand(OperandToReplace, PtrCastI); + return; + } +} + Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) { SmallVector Types = {I.getType(), I.getOperand(0)->getType(), I.getOperand(1)->getType(), @@ -522,13 +640,18 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { for (auto &I : Worklist) { insertAssignPtrTypeIntrs(I); insertAssignTypeIntrs(I); + insertPtrCastInstr(I); } for (auto *I : Worklist) { TrackConstants = true; if (!I->getType()->isVoidTy() || isa(I)) IRB->SetInsertPoint(I->getNextNode()); + // Visitors return either the original/newly created instruction for further + // processing, nullptr otherwise. I = visit(*I); + if (!I) + continue; processInstrAfterVisit(I); } return true; diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp index 1bfce70fedc0..429b08e199cd 100644 --- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -125,12 +125,32 @@ static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR, SmallVector ToErase; for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : MBB) { - if (!isSpvIntrinsic(MI, Intrinsic::spv_bitcast)) + if (!isSpvIntrinsic(MI, Intrinsic::spv_bitcast) && + !isSpvIntrinsic(MI, Intrinsic::spv_ptrcast)) continue; assert(MI.getOperand(2).isReg()); MIB.setInsertPt(*MI.getParent(), MI); - MIB.buildBitcast(MI.getOperand(0).getReg(), MI.getOperand(2).getReg()); ToErase.push_back(&MI); + if (isSpvIntrinsic(MI, Intrinsic::spv_bitcast)) { + MIB.buildBitcast(MI.getOperand(0).getReg(), MI.getOperand(2).getReg()); + continue; + } + Register Def = MI.getOperand(0).getReg(); + Register Source = MI.getOperand(2).getReg(); + SPIRVType *BaseTy = GR->getOrCreateSPIRVType( + getMDOperandAsType(MI.getOperand(3).getMetadata(), 0), MIB); + SPIRVType *AssignedPtrType = GR->getOrCreateSPIRVPointerType( + BaseTy, MI, *MF.getSubtarget().getInstrInfo(), + addressSpaceToStorageClass(MI.getOperand(4).getImm())); + + // If the bitcast would be redundant, replace all uses with the source + // register. + if (GR->getSPIRVTypeForVReg(Source) == AssignedPtrType) { + MIB.getMRI()->replaceRegWith(Def, Source); + } else { + GR->assignSPIRVTypeToVReg(AssignedPtrType, Def, MF); + MIB.buildBitcast(Def, Source); + } } } for (MachineInstr *MI : ToErase) diff --git a/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll b/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll index 8d12414c66a8..323afec7f35f 100644 --- a/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll +++ b/llvm/test/CodeGen/SPIRV/AtomicCompareExchange.ll @@ -10,47 +10,40 @@ ; CHECK-SPIRV-DAG: %[[#Struct:]] = OpTypeStruct %[[#Int]] %[[#Bool]] ; CHECK-SPIRV-DAG: %[[#UndefStruct:]] = OpUndef %[[#Struct]] -; CHECK-SPIRV: %[[#Pointer:]] = OpFunctionParameter %[[#]] -; CHECK-SPIRV: %[[#Value_ptr:]] = OpFunctionParameter %[[#]] -; CHECK-SPIRV: %[[#Comparator:]] = OpFunctionParameter %[[#]] - -; CHECK-SPIRV: %[[#Value:]] = OpLoad %[[#Int]] %[[#Value_ptr]] -; CHECK-SPIRV: %[[#Res:]] = OpAtomicCompareExchange %[[#Int]] %[[#Pointer]] %[[#MemScope_Device]] -; CHECK-SPIRV-SAME: %[[#MemSemEqual_SeqCst]] %[[#MemSemUnequal_Acquire]] %[[#Value]] %[[#Comparator]] +; CHECK-SPIRV: %[[#Value:]] = OpLoad %[[#Int]] %[[#Value_ptr:]] +; CHECK-SPIRV: %[[#Res:]] = OpAtomicCompareExchange %[[#Int]] %[[#Pointer:]] %[[#MemScope_Device]] +; CHECK-SPIRV-SAME: %[[#MemSemEqual_SeqCst]] %[[#MemSemUnequal_Acquire]] %[[#Value]] %[[#Comparator:]] ; CHECK-SPIRV: %[[#Success:]] = OpIEqual %[[#]] %[[#Res]] %[[#Comparator]] ; CHECK-SPIRV: %[[#Composite_0:]] = OpCompositeInsert %[[#Struct]] %[[#Res]] %[[#UndefStruct]] 0 ; CHECK-SPIRV: %[[#Composite_1:]] = OpCompositeInsert %[[#Struct]] %[[#Success]] %[[#Composite_0]] 1 ; CHECK-SPIRV: %[[#]] = OpCompositeExtract %[[#Bool]] %[[#Composite_1]] 1 -define dso_local spir_func void @test(i32* %ptr, i32* %value_ptr, i32 %comparator) local_unnamed_addr { +define dso_local spir_func void @test(ptr %ptr, ptr %value_ptr, i32 %comparator) local_unnamed_addr { entry: - %0 = load i32, i32* %value_ptr, align 4 - %1 = cmpxchg i32* %ptr, i32 %comparator, i32 %0 seq_cst acquire + %0 = load i32, ptr %value_ptr, align 4 + %1 = cmpxchg ptr %ptr, i32 %comparator, i32 %0 seq_cst acquire %2 = extractvalue { i32, i1 } %1, 1 br i1 %2, label %cmpxchg.continue, label %cmpxchg.store_expected cmpxchg.store_expected: ; preds = %entry %3 = extractvalue { i32, i1 } %1, 0 - store i32 %3, i32* %value_ptr, align 4 + store i32 %3, ptr %value_ptr, align 4 br label %cmpxchg.continue cmpxchg.continue: ; preds = %cmpxchg.store_expected, %entry ret void } -; CHECK-SPIRV: %[[#Ptr:]] = OpFunctionParameter %[[#]] -; CHECK-SPIRV: %[[#Store_ptr:]] = OpFunctionParameter %[[#]] - -; CHECK-SPIRV: %[[#Res_1:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr]] %[[#MemScope_Device]] +; CHECK-SPIRV: %[[#Res_1:]] = OpAtomicCompareExchange %[[#Int]] %[[#Ptr:]] %[[#MemScope_Device]] ; CHECK-SPIRV-SAME: %[[#MemSemEqual_SeqCst]] %[[#MemSemUnequal_Acquire]] %[[#Constant_456]] %[[#Constant_128]] ; CHECK-SPIRV: %[[#Success_1:]] = OpIEqual %[[#]] %[[#Res_1]] %[[#Constant_128]] ; CHECK-SPIRV: %[[#Composite:]] = OpCompositeInsert %[[#Struct]] %[[#Res_1]] %[[#UndefStruct]] 0 ; CHECK-SPIRV: %[[#Composite_1:]] = OpCompositeInsert %[[#Struct]] %[[#Success_1]] %[[#Composite]] 1 -; CHECK-SPIRV: OpStore %[[#Store_ptr]] %[[#Composite_1]] +; CHECK-SPIRV: OpStore %[[#Store_ptr:]] %[[#Composite_1]] -define dso_local spir_func void @test2(i32* %ptr, {i32, i1}* %store_ptr) local_unnamed_addr { +define dso_local spir_func void @test2(ptr %ptr, ptr %store_ptr) local_unnamed_addr { entry: - %0 = cmpxchg i32* %ptr, i32 128, i32 456 seq_cst acquire - store { i32, i1 } %0, { i32, i1 }* %store_ptr, align 4 + %0 = cmpxchg ptr %ptr, i32 128, i32 456 seq_cst acquire + store { i32, i1 } %0, ptr %store_ptr, align 4 ret void } diff --git a/llvm/test/CodeGen/SPIRV/function/alloca-load-store.ll b/llvm/test/CodeGen/SPIRV/function/alloca-load-store.ll index 5e76eb11de63..c64a708b28eb 100644 --- a/llvm/test/CodeGen/SPIRV/function/alloca-load-store.ll +++ b/llvm/test/CodeGen/SPIRV/function/alloca-load-store.ll @@ -1,17 +1,16 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s -target triple = "spirv32-unknown-unknown" - ; CHECK-DAG: OpName %[[#BAR:]] "bar" ; CHECK-DAG: OpName %[[#FOO:]] "foo" ; CHECK-DAG: OpName %[[#GOO:]] "goo" ; CHECK-DAG: %[[#CHAR:]] = OpTypeInt 8 ; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 -; CHECK-DAG: %[[#STACK_PTR:]] = OpTypePointer Function %[[#INT]] -; CHECK-DAG: %[[#GLOBAL_PTR:]] = OpTypePointer CrossWorkgroup %[[#CHAR]] +; CHECK-DAG: %[[#STACK_PTR_INT:]] = OpTypePointer Function %[[#INT]] +; CHECK-DAG: %[[#GLOBAL_PTR_INT:]] = OpTypePointer CrossWorkgroup %[[#INT]] +; CHECK-DAG: %[[#GLOBAL_PTR_CHAR:]] = OpTypePointer CrossWorkgroup %[[#CHAR]] ; CHECK-DAG: %[[#FN1:]] = OpTypeFunction %[[#INT]] %[[#INT]] -; CHECK-DAG: %[[#FN2:]] = OpTypeFunction %[[#INT]] %[[#INT]] %[[#GLOBAL_PTR]] +; CHECK-DAG: %[[#FN2:]] = OpTypeFunction %[[#INT]] %[[#INT]] %[[#GLOBAL_PTR_CHAR]] define i32 @bar(i32 %a) { %p = alloca i32 @@ -23,7 +22,7 @@ define i32 @bar(i32 %a) { ; CHECK: %[[#BAR]] = OpFunction %[[#INT]] None %[[#FN1]] ; CHECK: %[[#A:]] = OpFunctionParameter %[[#INT]] ; CHECK: OpLabel -; CHECK: %[[#P:]] = OpVariable %[[#STACK_PTR]] Function +; CHECK: %[[#P:]] = OpVariable %[[#STACK_PTR_INT]] Function ; CHECK: OpStore %[[#P]] %[[#A]] ; CHECK: %[[#B:]] = OpLoad %[[#INT]] %[[#P]] ; CHECK: OpReturnValue %[[#B]] @@ -40,7 +39,7 @@ define i32 @foo(i32 %a) { ; CHECK: %[[#FOO]] = OpFunction %[[#INT]] None %[[#FN1]] ; CHECK: %[[#A:]] = OpFunctionParameter %[[#INT]] ; CHECK: OpLabel -; CHECK: %[[#P:]] = OpVariable %[[#STACK_PTR]] Function +; CHECK: %[[#P:]] = OpVariable %[[#STACK_PTR_INT]] Function ; CHECK: OpStore %[[#P]] %[[#A]] Volatile ; CHECK: %[[#B:]] = OpLoad %[[#INT]] %[[#P]] Volatile ; CHECK: OpReturnValue %[[#B]] @@ -48,7 +47,7 @@ define i32 @foo(i32 %a) { ;; Test load and store in global address space. -define i32 @goo(i32 %a, i32 addrspace(1)* %p) { +define i32 @goo(i32 %a, ptr addrspace(1) %p) { store i32 %a, i32 addrspace(1)* %p %b = load i32, i32 addrspace(1)* %p ret i32 %b @@ -56,9 +55,10 @@ define i32 @goo(i32 %a, i32 addrspace(1)* %p) { ; CHECK: %[[#GOO]] = OpFunction %[[#INT]] None %[[#FN2]] ; CHECK: %[[#A:]] = OpFunctionParameter %[[#INT]] -; CHECK: %[[#P:]] = OpFunctionParameter %[[#GLOBAL_PTR]] +; CHECK: %[[#P:]] = OpFunctionParameter %[[#GLOBAL_PTR_CHAR]] ; CHECK: OpLabel -; CHECK: OpStore %[[#P]] %[[#A]] -; CHECK: %[[#B:]] = OpLoad %[[#INT]] %[[#P]] +; CHECK: %[[#C:]] = OpBitcast %[[#GLOBAL_PTR_INT]] %[[#P]] +; CHECK: OpStore %[[#C]] %[[#A]] +; CHECK: %[[#B:]] = OpLoad %[[#INT]] %[[#C]] ; CHECK: OpReturnValue %[[#B]] ; CHECK: OpFunctionEnd diff --git a/llvm/test/CodeGen/SPIRV/instructions/undef-nested-composite-store.ll b/llvm/test/CodeGen/SPIRV/instructions/undef-nested-composite-store.ll index 9fa94b848463..5fa4a7a93ee0 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/undef-nested-composite-store.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/undef-nested-composite-store.ll @@ -9,7 +9,8 @@ ; CHECK: %[[#]] = OpFunction %[[#]] None %[[#]] ; CHECK-NEXT: %[[#PTR:]] = OpFunctionParameter %[[#]] ; CHECK-NEXT: %[[#]] = OpLabel -; CHECK-NEXT: OpStore %[[#PTR]] %[[#UNDEF]] Aligned 4 +; CHECK-NEXT: %[[#BC:]] = OpBitcast %[[#]] %[[#PTR]] +; CHECK-NEXT: OpStore %[[#BC]] %[[#UNDEF]] Aligned 4 ; CHECK-NEXT: OpReturn ; CHECK-NEXT: OpFunctionEnd diff --git a/llvm/test/CodeGen/SPIRV/instructions/undef-simple-composite-store.ll b/llvm/test/CodeGen/SPIRV/instructions/undef-simple-composite-store.ll index 6a274732417d..24dad10382ad 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/undef-simple-composite-store.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/undef-simple-composite-store.ll @@ -8,7 +8,8 @@ ; CHECK: %[[#]] = OpFunction %[[#]] None %[[#]] ; CHECK-NEXT: %[[#PTR:]] = OpFunctionParameter %[[#]] ; CHECK-NEXT: %[[#]] = OpLabel -; CHECK-NEXT: OpStore %[[#PTR]] %[[#UNDEF]] Aligned 4 +; CHECK-NEXT: %[[#BC:]] = OpBitcast %[[#]] %[[#PTR]] +; CHECK-NEXT: OpStore %[[#BC]] %[[#UNDEF]] Aligned 4 ; CHECK-NEXT: OpReturn ; CHECK-NEXT: OpFunctionEnd diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memset.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memset.ll index 3d7d9fbf9075..e7a986980f25 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memset.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/memset.ll @@ -25,9 +25,7 @@ ; CHECK: %[[#VarNull:]] = OpVariable %[[#]] UniformConstant %[[#ConstNull]] ; CHECK-DAG: %[[#Int8PtrConst:]] = OpTypePointer UniformConstant %[[#Int8]] -; CHECK: %[[#Target:]] = OpBitcast %[[#Int8Ptr]] %[[#]] -; CHECK: %[[#Source:]] = OpBitcast %[[#Int8PtrConst]] %[[#VarNull]] -; CHECK: OpCopyMemorySized %[[#Target]] %[[#Source]] %[[#Const12]] Aligned 4 +; CHECK: OpCopyMemorySized %[[#Target:]] %[[#Source:]] %[[#Const12]] Aligned 4 ; CHECK: %[[#SourceComp:]] = OpBitcast %[[#Int8PtrConst]] %[[#VarComp]] ; CHECK: OpCopyMemorySized %[[#]] %[[#SourceComp]] %[[#Const4]] Aligned 4 diff --git a/llvm/test/CodeGen/SPIRV/opaque_pointers.ll b/llvm/test/CodeGen/SPIRV/opaque_pointers.ll index aaddea137272..30c9d8dc774e 100644 --- a/llvm/test/CodeGen/SPIRV/opaque_pointers.ll +++ b/llvm/test/CodeGen/SPIRV/opaque_pointers.ll @@ -1,19 +1,23 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK ; CHECK-DAG: %[[#Int8Ty:]] = OpTypeInt 8 0 -; CHECK-DAG: %[[#PtrTy:]] = OpTypePointer Function %[[#Int8Ty]] -; CHECK-DAG: %[[#Int64Ty:]] = OpTypeInt 64 0 -; CHECK-DAG: %[[#FTy:]] = OpTypeFunction %[[#Int64Ty]] %[[#PtrTy]] +; CHECK-DAG: %[[#PtrInt8Ty:]] = OpTypePointer Function %[[#Int8Ty]] ; CHECK-DAG: %[[#Int32Ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#PtrInt32Ty:]] = OpTypePointer Function %[[#Int32Ty]] +; CHECK-DAG: %[[#Int64Ty:]] = OpTypeInt 64 0 +; CHECK-DAG: %[[#PtrInt64Ty:]] = OpTypePointer Function %[[#Int64Ty]] +; CHECK-DAG: %[[#FTy:]] = OpTypeFunction %[[#Int64Ty]] %[[#PtrInt8Ty]] ; CHECK-DAG: %[[#Const:]] = OpConstant %[[#Int32Ty]] 0 ; CHECK: OpFunction %[[#Int64Ty]] None %[[#FTy]] -; CHECK: %[[#Parm:]] = OpFunctionParameter %[[#PtrTy]] -; CHECK: OpStore %[[#Parm]] %[[#Const]] Aligned 4 -; CHECK: %[[#Res:]] = OpLoad %[[#Int64Ty]] %[[#Parm]] Aligned 8 +; CHECK: %[[#Parm:]] = OpFunctionParameter %[[#PtrInt8Ty]] +; CHECK-DAG: %[[#Bitcast1:]] = OpBitcast %[[#PtrInt32Ty]] %[[#Parm]] +; CHECK: OpStore %[[#Bitcast1]] %[[#Const]] Aligned 4 +; CHECK-DAG: %[[#Bitcast2:]] = OpBitcast %[[#PtrInt64Ty]] %[[#Parm]] +; CHECK: %[[#Res:]] = OpLoad %[[#Int64Ty]] %[[#Bitcast2]] Aligned 4 ; CHECK: OpReturnValue %[[#Res]] define i64 @test(ptr %p) { - store i32 0, ptr %p - %v = load i64, ptr %p + store i32 0, ptr %p, align 4 + %v = load i64, ptr %p, align 4 ret i64 %v } diff --git a/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll b/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll index 127804671cee..43d6238360f6 100644 --- a/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll +++ b/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll @@ -1,10 +1,7 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s ; CHECK: OpEntryPoint Kernel %[[#test_func:]] "test" -; CHECK: OpName %[[#outOffsets:]] "outOffsets" -; CHECK: OpName %[[#test_func]] "test" -; CHECK: OpName %[[#f2_decl:]] "BuiltInGlobalOffset" -; CHECK: OpDecorate %[[#f2_decl]] LinkageAttributes "BuiltInGlobalOffset" Import +; CHECK: OpDecorate %[[#f2_decl:]] LinkageAttributes "BuiltInGlobalOffset" Import ; CHECK: %[[#int_ty:]] = OpTypeInt 8 0 ; CHECK: %[[#void_ty:]] = OpTypeVoid ; CHECK: %[[#iptr_ty:]] = OpTypePointer CrossWorkgroup %[[#int_ty]] @@ -26,14 +23,13 @@ ; CHECK-NOT: %[[#vec_ty]] = OpFunction ; CHECK-NOT: %[[#func2_ty]] = OpFunction ; CHECK-NOT: %[[#f2_decl]] = OpFunction -; CHECK: %[[#outOffsets]] = OpFunctionParameter %[[#iptr_ty]] define spir_kernel void @test(i32 addrspace(1)* %outOffsets) { entry: %0 = call spir_func <3 x i64> @BuiltInGlobalOffset() #1 %call = extractelement <3 x i64> %0, i32 0 %conv = trunc i64 %call to i32 -; CHECK: %[[#i1:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets]] +; CHECK: %[[#i1:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets:]] ; CHECK: OpStore %[[#i1:]] %[[#]] Aligned 4 %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %outOffsets, i64 0 store i32 %conv, i32 addrspace(1)* %arrayidx, align 4 diff --git a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll new file mode 100644 index 000000000000..9e136ce88746 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type-deduction-no-bitcast-to-generic.ll @@ -0,0 +1,19 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s + +; CHECK-DAG: %[[#IMAGE:]] = OpTypeImage %2 2D 0 0 0 0 Unknown ReadOnly + +; CHECK: %[[#PARAM:]] = OpFunctionParameter %[[#IMAGE:]] +; CHECK-NOT: OpBitcast +; CHECK: %[[#]] = OpImageQuerySizeLod %[[#]] %[[#PARAM]] %[[#]] + +define spir_kernel void @test(ptr addrspace(1) %srcimg) #0 !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_type_qual !4 !kernel_arg_base_type !3 { + %call = call spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1) %srcimg) + ret void +} + +declare spir_func <2 x i32> @_Z13get_image_dim14ocl_image2d_ro(ptr addrspace(1)) + +!1 = !{i32 1} +!2 = !{!"read_only"} +!3 = !{!"image2d_t"} +!4 = !{!""} diff --git a/llvm/test/CodeGen/SPIRV/pointers/two-bitcast-users.ll b/llvm/test/CodeGen/SPIRV/pointers/two-bitcast-users.ll new file mode 100644 index 000000000000..f4ea710fa043 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/two-bitcast-users.ll @@ -0,0 +1,19 @@ +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s + +; CHECK-DAG: %[[#CHAR:]] = OpTypeInt 8 +; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 +; CHECK-DAG: %[[#GLOBAL_PTR_INT:]] = OpTypePointer CrossWorkgroup %[[#INT]] +; CHECK-DAG: %[[#GLOBAL_PTR_CHAR:]] = OpTypePointer CrossWorkgroup %[[#CHAR]] + +define i32 @foo(i32 %a, ptr addrspace(1) %p) { + store i32 %a, i32 addrspace(1)* %p + %b = load i32, i32 addrspace(1)* %p + ret i32 %b +} + +; CHECK: %[[#A:]] = OpFunctionParameter %[[#INT]] +; CHECK: %[[#P:]] = OpFunctionParameter %[[#GLOBAL_PTR_CHAR]] +; CHECK: %[[#C:]] = OpBitcast %[[#GLOBAL_PTR_INT]] %[[#P]] +; CHECK: OpStore %[[#C]] %[[#A]] +; CHECK: %[[#B:]] = OpLoad %[[#INT]] %[[#C]] +; CHECK-NOT: %[[#B:]] = OpLoad %[[#INT]] %[[#P]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll b/llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll new file mode 100644 index 000000000000..8998329ea64f --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/two-subsequent-bitcasts.ll @@ -0,0 +1,17 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s + +; CHECK-DAG: %[[#float:]] = OpTypeFloat 32 +; CHECK-DAG: %[[#pointer:]] = OpTypePointer CrossWorkgroup %[[#float]] +; CHECK: %[[#A:]] = OpFunctionParameter %[[#]] + +define void @foo(float addrspace(1)* %A, i32 %B) { + %cmp = icmp sgt i32 %B, 0 + %conv = uitofp i1 %cmp to float +; CHECK: %[[#utof_res:]] = OpConvertUToF %[[#float]] %[[#]] +; CHECK: %[[#bitcast:]] = OpBitcast %[[#pointer]] %[[#A]] +; CHECK: OpStore %[[#bitcast]] %[[#utof_res]] + %BC1 = bitcast float addrspace(1)* %A to i32 addrspace(1)* + %BC2 = bitcast i32 addrspace(1)* %BC1 to float addrspace(1)* + store float %conv, float addrspace(1)* %BC2, align 4; + ret void +} diff --git a/llvm/test/CodeGen/SPIRV/sitofp-with-bool.ll b/llvm/test/CodeGen/SPIRV/sitofp-with-bool.ll index 4dda0aadced4..06f27e2cb1f9 100644 --- a/llvm/test/CodeGen/SPIRV/sitofp-with-bool.ll +++ b/llvm/test/CodeGen/SPIRV/sitofp-with-bool.ll @@ -1,9 +1,11 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s -; CHECK: %[[#int_32:]] = OpTypeInt 32 0 -; CHECK: %[[#bool:]] = OpTypeBool -; CHECK: %[[#zero:]] = OpConstant %[[#int_32]] 0 -; CHECK: %[[#one:]] = OpConstant %[[#int_32]] 1 +; CHECK-DAG: %[[#int_32:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#float:]] = OpTypeFloat 32 +; CHECK-DAG: %[[#bool:]] = OpTypeBool +; CHECK-DAG: %[[#zero:]] = OpConstant %[[#int_32]] 0 +; CHECK-DAG: %[[#one:]] = OpConstant %[[#int_32]] 1 +; CHECK-DAG: %[[#ptr:]] = OpTypePointer CrossWorkgroup %[[#float]] ; CHECK: OpFunction ; CHECK: %[[#A:]] = OpFunctionParameter %[[#]] @@ -11,9 +13,10 @@ ; CHECK: %[[#cmp_res:]] = OpSGreaterThan %[[#bool]] %[[#B]] %[[#zero]] ; CHECK: %[[#select_res:]] = OpSelect %[[#int_32]] %[[#cmp_res]] %[[#one]] %[[#zero]] ; CHECK: %[[#stof_res:]] = OpConvertSToF %[[#]] %[[#select_res]] -; CHECK: OpStore %[[#A]] %[[#stof_res]] +; CHECK: %[[#bitcast:]] = OpBitcast %[[#ptr]] %[[#A]] +; CHECK: OpStore %[[#bitcast]] %[[#stof_res]] -define dso_local spir_kernel void @K(float addrspace(1)* nocapture %A, i32 %B) local_unnamed_addr { +define dso_local spir_kernel void @K(ptr addrspace(1) nocapture %A, i32 %B) local_unnamed_addr { entry: %cmp = icmp sgt i32 %B, 0 %conv = sitofp i1 %cmp to float diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll index 9252e264cec8..c98fef3631e0 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpPhi_ArgumentsPlaceholders.ll @@ -13,6 +13,7 @@ ;; } ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; XFAIL: * %struct.Node = type { %struct.Node.0 addrspace(1)* } %struct.Node.0 = type opaque diff --git a/llvm/test/CodeGen/SPIRV/transcoding/spec_const.ll b/llvm/test/CodeGen/SPIRV/transcoding/spec_const.ll index 9fd746088524..c47dccb35e14 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/spec_const.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/spec_const.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; XFAIL: * ; CHECK-SPIRV-NOT: OpCapability Matrix ; CHECK-SPIRV-NOT: OpCapability Shader diff --git a/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll b/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll index 3c45fdd0481d..a07af76c2232 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll @@ -19,10 +19,8 @@ ; CHECK-SPIRV: %[[#arr:]] = OpVariable %[[#i32x3_ptr]] Function ; CHECK-SPIRV: %[[#arr2:]] = OpVariable %[[#i32x3_ptr]] Function -; CHECK-SPIRV: %[[#arr_i8_ptr:]] = OpBitcast %[[#i8_ptr]] %[[#arr]] -; CHECK-SPIRV: OpCopyMemorySized %[[#arr_i8_ptr]] %[[#test_arr]] %[[#twelve]] Aligned 4 -; CHECK-SPIRV: %[[#arr2_i8_ptr:]] = OpBitcast %[[#i8_ptr]] %[[#arr2]] -; CHECK-SPIRV: OpCopyMemorySized %[[#arr2_i8_ptr]] %[[#test_arr2]] %[[#twelve]] Aligned 4 +; CHECK-SPIRV: OpCopyMemorySized %[[#arr]] %[[#test_arr]] %[[#twelve]] Aligned 4 +; CHECK-SPIRV: OpCopyMemorySized %[[#arr2]] %[[#test_arr2]] %[[#twelve]] Aligned 4 @__const.test.arr = private unnamed_addr addrspace(2) constant [3 x i32] [i32 1, i32 2, i32 3], align 4 diff --git a/llvm/test/CodeGen/SPIRV/uitofp-with-bool.ll b/llvm/test/CodeGen/SPIRV/uitofp-with-bool.ll index 2fe0dc91e7b4..8a3c2853d991 100644 --- a/llvm/test/CodeGen/SPIRV/uitofp-with-bool.ll +++ b/llvm/test/CodeGen/SPIRV/uitofp-with-bool.ll @@ -66,6 +66,7 @@ ; SPV-DAG: %[[#ones_16:]] = OpConstantComposite %[[#vec_16]] %[[#one_16]] %[[#one_16]] ; SPV-DAG: %[[#ones_32:]] = OpConstantComposite %[[#vec_32]] %[[#one_32]] %[[#one_32]] ; SPV-DAG: %[[#ones_64:]] = OpConstantComposite %[[#vec_64]] %[[#one_64]] %[[#one_64]] +; SPV-DAG: %[[#pointer:]] = OpTypePointer CrossWorkgroup %[[#float]] ; SPV-DAG: OpFunction ; SPV-DAG: %[[#A:]] = OpFunctionParameter %[[#]] @@ -81,7 +82,8 @@ entry: ; SPV-DAG: %[[#select_res:]] = OpSelect %[[#int_32]] %[[#cmp_res]] %[[#one_32]] %[[#zero_32]] ; SPV-DAG: %[[#utof_res:]] = OpConvertUToF %[[#float]] %[[#select_res]] %conv = uitofp i1 %cmp to float -; SPV-DAG: OpStore %[[#A]] %[[#utof_res]] +; SPV-DAG: %[[#bitcast:]] = OpBitcast %[[#pointer]] %[[#A]] +; SPV-DAG: OpStore %[[#bitcast]] %[[#utof_res]] store float %conv, float addrspace(1)* %A, align 4; ; SPV-DAG: %[[#s1]] = OpSelect %[[#int_8]] %[[#i1s]] %[[#mone_8]] %[[#zero_8]] -- GitLab From f5fd1836836e0d37dea61cc842199713cc0e2fc4 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Fri, 5 Jan 2024 11:16:53 +0800 Subject: [PATCH 066/728] [NFC] [C++20] [Modules] Remove pr60085.cppm with deprecated practice See https://github.com/llvm/llvm-project/issues/60085 for the complete story. Previously I thought the problem got fixed surprisingly. But it is not true. I just tested it with a deprecated method. My bad. Then the deprecated style should be removed and the proper style can't work. So I'll remove the test and reopen that issue to look into it. --- clang/test/Modules/pr60085.cppm | 98 --------------------------------- 1 file changed, 98 deletions(-) delete mode 100644 clang/test/Modules/pr60085.cppm diff --git a/clang/test/Modules/pr60085.cppm b/clang/test/Modules/pr60085.cppm deleted file mode 100644 index fba601206404..000000000000 --- a/clang/test/Modules/pr60085.cppm +++ /dev/null @@ -1,98 +0,0 @@ -// RUN: rm -rf %t -// RUN: mkdir %t -// RUN: split-file %s %t -// -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/d.cppm \ -// RUN: -emit-module-interface -o %t/d.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/c.cppm \ -// RUN: -emit-module-interface -o %t/c.pcm -fmodule-file=%t/d.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.cppm \ -// RUN: -emit-module-interface -o %t/b.pcm -fmodule-file=%t/d.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.cppm \ -// RUN: -emit-module-interface -o %t/a.pcm -fmodule-file=%t/d.pcm \ -// RUN: -fmodule-file=%t/c.pcm -fmodule-file=%t/b.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.pcm \ -// RUN: -S -emit-llvm -disable-llvm-passes -o - | FileCheck %t/a.cppm -// -// Use -fmodule-file== -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/d.cppm \ -// RUN: -emit-module-interface -o %t/d.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/c.cppm \ -// RUN: -emit-module-interface -o %t/c.pcm -fmodule-file=%t/d.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/b.cppm \ -// RUN: -emit-module-interface -o %t/b.pcm -fmodule-file=%t/d.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.cppm \ -// RUN: -emit-module-interface -o %t/a.pcm -fmodule-file=%t/d.pcm \ -// RUN: -fmodule-file=%t/c.pcm -fmodule-file=%t/b.pcm -// RUN: %clang_cc1 -std=c++20 -triple %itanium_abi_triple %t/a.pcm \ -// RUN: -S -emit-llvm -disable-llvm-passes -o - | FileCheck %t/a.cppm - -//--- d.cppm -export module d; - -export template -struct integer { - using type = int; - - static constexpr auto value() { - return 0; - } - - friend constexpr void f(integer const x) { - x.value(); - } -}; - -export constexpr void ddd(auto const value) { - f(value); -} - - -template -constexpr auto dd = T(); - -export template -constexpr auto d() { - dd; -} - -//--- c.cppm -export module c; - -import d; - -template -auto cc = T(); - -auto c() { - cc>; - integer().value(); -} - -//--- b.cppm -export module b; - -import d; - -auto b() { - integer::type; -} - -//--- a.cppm -export module a; - -import b; -import c; -import d; - -constexpr void aa() { - d>(); - ddd(integer()); -} - -export extern "C" void a() { - aa(); -} - -// Checks that we emit the IR successfully. -// CHECK: define{{.*}}@a( -- GitLab From 054b5fc0fd41bcbadcc6967c39a5f6bb151bdcd1 Mon Sep 17 00:00:00 2001 From: Benoit Jacob Date: Thu, 4 Jan 2024 22:39:40 -0500 Subject: [PATCH 067/728] X86: add some missing lowerings for shuffles on `bf16` element type. (#76076) Some shuffles with `bf16` as element type were running into a `llvm_unreachable`. Key to reproducing was to chain two shuffles. ```llvm define <2 x bfloat> @shuffle_chained_v32bf16_v2bf16(<32 x bfloat> %a) { %s = shufflevector <32 x bfloat> %a, <32 x bfloat> zeroinitializer, <32 x i32> %s2 = shufflevector <32 x bfloat> %s, <32 x bfloat> zeroinitializer, <2 x i32> ret <2 x bfloat> %s2 } ``` This was hitting this UNREACHABLE: ``` Not a valid 512-bit x86 vector type! UNREACHABLE executed at /home/benoit/iree/third_party/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp:17124! PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace. Stack dump: 0. Program arguments: /home/benoit/mlir-build/bin/llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512bw,+avx512bf16 1. Running pass 'Function Pass Manager' on module ''. 2. Running pass 'X86 DAG->DAG Instruction Selection' on function '@shuffle_chained_v32bf16_v2bf16' ``` --- llvm/lib/Target/X86/X86ISelLowering.cpp | 11 +++- .../avx512-shuffles/shuffle-chained-bf16.ll | 63 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/X86/avx512-shuffles/shuffle-chained-bf16.ll diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index fe3ba2ae2917..a72b14c2e2aa 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -14369,6 +14369,13 @@ static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) { + if (VT == MVT::v8bf16) { + V1 = DAG.getBitcast(MVT::v8i16, V1); + V2 = DAG.getBitcast(MVT::v8i16, V2); + return DAG.getBitcast(VT, + DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask)); + } + switch (VT.SimpleTy) { case MVT::v2i64: return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); @@ -17096,14 +17103,14 @@ static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef Mask, return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false); } - if (VT == MVT::v32f16) { + if (VT == MVT::v32f16 || VT == MVT::v32bf16) { if (!Subtarget.hasBWI()) return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false); V1 = DAG.getBitcast(MVT::v32i16, V1); V2 = DAG.getBitcast(MVT::v32i16, V2); - return DAG.getBitcast(MVT::v32f16, + return DAG.getBitcast(VT, DAG.getVectorShuffle(MVT::v32i16, DL, V1, V2, Mask)); } diff --git a/llvm/test/CodeGen/X86/avx512-shuffles/shuffle-chained-bf16.ll b/llvm/test/CodeGen/X86/avx512-shuffles/shuffle-chained-bf16.ll new file mode 100644 index 000000000000..99d6049fc1d8 --- /dev/null +++ b/llvm/test/CodeGen/X86/avx512-shuffles/shuffle-chained-bf16.ll @@ -0,0 +1,63 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512bw,+avx512bf16 | FileCheck %s + +target triple = "x86_64-unknown-linux-gnu" + +define <2 x bfloat> @shuffle_chained_v32bf16_v2bf16(<32 x bfloat> %a) { +; CHECK-LABEL: shuffle_chained_v32bf16_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset %rbp, -16 +; CHECK-NEXT: movq %rsp, %rbp +; CHECK-NEXT: .cfi_def_cfa_register %rbp +; CHECK-NEXT: andq $-64, %rsp +; CHECK-NEXT: subq $128, %rsp +; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [0,16,0,16,0,16,0,16] +; CHECK-NEXT: vpermw %zmm0, %zmm1, %zmm0 +; CHECK-NEXT: vmovdqa64 %zmm0, (%rsp) +; CHECK-NEXT: vmovaps (%rsp), %xmm0 +; CHECK-NEXT: movq %rbp, %rsp +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: .cfi_def_cfa %rsp, 8 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %s = shufflevector <32 x bfloat> %a, <32 x bfloat> zeroinitializer, <32 x i32> + %s2 = shufflevector <32 x bfloat> %s, <32 x bfloat> zeroinitializer, <2 x i32> + ret <2 x bfloat> %s2 +} + +define <2 x bfloat> @shuffle_chained_v16bf16(<16 x bfloat> %a) { +; CHECK-LABEL: shuffle_chained_v16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset %rbp, -16 +; CHECK-NEXT: movq %rsp, %rbp +; CHECK-NEXT: .cfi_def_cfa_register %rbp +; CHECK-NEXT: andq $-32, %rsp +; CHECK-NEXT: subq $96, %rsp +; CHECK-NEXT: vmovaps %ymm0, (%rsp) +; CHECK-NEXT: vmovdqa (%rsp), %xmm0 +; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] +; CHECK-NEXT: vmovdqa %ymm0, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0 +; CHECK-NEXT: movq %rbp, %rsp +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: .cfi_def_cfa %rsp, 8 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %s = shufflevector <16 x bfloat> %a, <16 x bfloat> zeroinitializer, <16 x i32> + %s2 = shufflevector <16 x bfloat> %s, <16 x bfloat> zeroinitializer, <2 x i32> + ret <2 x bfloat> %s2 +} + +define <2 x bfloat> @shuffle_chained_v8bf16(<8 x bfloat> %a) { +; CHECK-LABEL: shuffle_chained_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15] +; CHECK-NEXT: retq + %s = shufflevector <8 x bfloat> %a, <8 x bfloat> zeroinitializer, <8 x i32> + %s2 = shufflevector <8 x bfloat> %s, <8 x bfloat> zeroinitializer, <2 x i32> + ret <2 x bfloat> %s2 +} -- GitLab From 52d1397e38ee88b170585c9c824d08e6975890ca Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Fri, 5 Jan 2024 12:05:23 +0800 Subject: [PATCH 068/728] [LoongArch] Fix -Wunused-variable in LoongArchExpandPseudoInsts.cpp (NFC) llvm-project/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp:480:20: error: unused variable 'MF' [-Werror,-Wunused-variable] MachineFunction *MF = MBB.getParent(); ^ 1 error generated. --- llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp index f977f176066a..ad39658f698e 100644 --- a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp @@ -477,12 +477,11 @@ bool LoongArchExpandPseudo::expandLargeAddressLoad( break; } - MachineFunction *MF = MBB.getParent(); MachineInstr &MI = *MBBI; DebugLoc DL = MI.getDebugLoc(); Register ScratchReg = LoongArch::R20; // $t8 - assert(MF->getSubtarget().is64Bit() && + assert(MBB.getParent()->getSubtarget().is64Bit() && "Large code model requires LA64"); auto Part1 = BuildMI(MBB, MBBI, DL, TII->get(LoongArch::PCALAU12I), DestReg); -- GitLab From 16dc82122bee915d122a68a4f1680ab810012906 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 4 Jan 2024 21:46:02 -0800 Subject: [PATCH 069/728] [RISCV] Remove isGPRF64AsFPR and isGPRPF64AsFPR functions from AsmParser. NFC These are identical to isGPRAsFPR. By overriding the PredicateMethod on the AsmOperands in tblgen we can share a single function. --- llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp | 4 ---- llvm/lib/Target/RISCV/RISCVInstrInfoD.td | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 4759aa951664..23c2b63c8c83 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -466,10 +466,6 @@ public: bool isGPRAsFPR() const { return isGPR() && Reg.IsGPRAsFPR; } - bool isGPRF64AsFPR() const { return isGPR() && Reg.IsGPRAsFPR; } - - bool isGPRPF64AsFPR() const { return isGPR() && Reg.IsGPRAsFPR; } - static bool evaluateConstantImm(const MCExpr *Expr, int64_t &Imm, RISCVMCExpr::VariantKind &VK) { if (auto *RE = dyn_cast(Expr)) { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td index 6af710049a9d..418421b2a556 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td @@ -36,11 +36,13 @@ def AddrRegImmINX : ComplexPattern; def GPRPF64AsFPR : AsmOperandClass { let Name = "GPRPF64AsFPR"; let ParserMethod = "parseGPRAsFPR"; + let PredicateMethod = "isGPRAsFPR"; let RenderMethod = "addRegOperands"; } def GPRF64AsFPR : AsmOperandClass { let Name = "GPRF64AsFPR"; + let PredicateMethod = "isGPRAsFPR"; let ParserMethod = "parseGPRAsFPR"; let RenderMethod = "addRegOperands"; } -- GitLab From 668165002543fd3a88413a5c2601774395bfd10e Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Fri, 5 Jan 2024 14:39:16 +0800 Subject: [PATCH 070/728] [InstCombine] Revert the `signed icmp -> unsigned icmp` canonicalization when folding `icmp Pred min|max(X, Y), Z` (#76685) This patch tries to flip the signedness of predicates when folding an unsigned icmp with a signed min/max. It will enable more optimizations as we canonicalizes a signed icmp into an unsigned icmp when both operands are known to have the same sign. Fixes #76672. Compile-time impact: http://llvm-compile-time-tracker.com/compare.php?from=949ec83eaf6fa6dbffb94c2ea9c0a4d5efdbd239&to=2deca1aea8a4e13609bab72c522a97d424f0fc2d&stat=instructions:u |stage1-O3|stage1-ReleaseThinLTO|stage1-ReleaseLTO-g|stage1-O0-g|stage2-O3|stage2-O0-g|stage2-clang| |--|--|--|--|--|--|--| |-0.00%|+0.01%|+0.05%|-0.12%|-0.01%|-0.03%|-0.00%| NOTE: We can flip the signedness of predicate if both operands are negative. But I don't see the benefit of handling these cases. --- .../InstCombine/InstCombineCompares.cpp | 12 ++- llvm/test/Transforms/InstCombine/smax-icmp.ll | 41 +++++++++ .../LoopVectorize/X86/float-induction-x86.ll | 4 +- .../X86/invariant-load-gather.ll | 43 +++++----- .../X86/invariant-store-vectorization.ll | 63 +++++++------- .../LoopVectorize/float-induction.ll | 8 +- .../Transforms/LoopVectorize/induction.ll | 38 ++++----- .../interleaved-accesses-pred-stores.ll | 83 +++++++++---------- .../LoopVectorize/interleaved-accesses.ll | 30 +++---- .../invariant-store-vectorization-2.ll | 15 ++-- .../invariant-store-vectorization.ll | 37 ++++----- .../Transforms/LoopVectorize/loop-scalars.ll | 8 +- .../Transforms/LoopVectorize/reduction.ll | 4 +- .../Transforms/LoopVectorize/vector-geps.ll | 14 ++-- 14 files changed, 215 insertions(+), 185 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 3875e59c3ede..2dc0b0b87f60 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -5047,8 +5047,16 @@ Instruction *InstCombinerImpl::foldICmpWithMinMax(Instruction &I, Value *Y = MinMax->getRHS(); if (ICmpInst::isSigned(Pred) && !MinMax->isSigned()) return nullptr; - if (ICmpInst::isUnsigned(Pred) && MinMax->isSigned()) - return nullptr; + if (ICmpInst::isUnsigned(Pred) && MinMax->isSigned()) { + // Revert the transform signed pred -> unsigned pred + // TODO: We can flip the signedness of predicate if both operands of icmp + // are negative. + if (isKnownNonNegative(Z, SQ.getWithInstruction(&I)) && + isKnownNonNegative(MinMax, SQ.getWithInstruction(&I))) { + Pred = ICmpInst::getFlippedSignednessPredicate(Pred); + } else + return nullptr; + } SimplifyQuery Q = SQ.getWithInstruction(&I); auto IsCondKnownTrue = [](Value *Val) -> std::optional { if (!Val) diff --git a/llvm/test/Transforms/InstCombine/smax-icmp.ll b/llvm/test/Transforms/InstCombine/smax-icmp.ll index 0ccbef34889f..022ec6ad4f34 100644 --- a/llvm/test/Transforms/InstCombine/smax-icmp.ll +++ b/llvm/test/Transforms/InstCombine/smax-icmp.ll @@ -804,4 +804,45 @@ end: ret void } +; Tests from PR76672 + +define i1 @test_smax_ugt(i32 %a) { +; CHECK-LABEL: @test_smax_ugt( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 1 +; CHECK-NEXT: ret i1 [[CMP]] +; +entry: + %cond.i = call i32 @llvm.smax.i32(i32 %a, i32 0) + %cmp = icmp ugt i32 %cond.i, 1 + ret i1 %cmp +} + +; Negative tests + +define i1 @test_smax_ugt_neg1(i32 %a) { +; CHECK-LABEL: @test_smax_ugt_neg1( +; CHECK-NEXT: entry: +; CHECK-NEXT: ret i1 false +; +entry: + %cond.i = call i32 @llvm.smax.i32(i32 %a, i32 0) + %cmp = icmp ugt i32 %cond.i, -5 + ret i1 %cmp +} + +define i1 @test_smax_ugt_neg2(i32 %a) { +; CHECK-LABEL: @test_smax_ugt_neg2( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[COND_I:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 -5) +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i32 [[COND_I]], 1 +; CHECK-NEXT: ret i1 [[CMP]] +; +entry: + %cond.i = call i32 @llvm.smax.i32(i32 %a, i32 -5) + %cmp = icmp ugt i32 %cond.i, 1 + ret i1 %cmp +} + + declare i32 @llvm.smax.i32(i32, i32) diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll index c8c791b30163..88186584c205 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll @@ -197,7 +197,7 @@ define double @external_use_with_fast_math(ptr %a, i64 %n) { ; AUTO_VEC-LABEL: @external_use_with_fast_math( ; AUTO_VEC-NEXT: entry: ; AUTO_VEC-NEXT: [[SMAX:%.*]] = tail call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; AUTO_VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16 +; AUTO_VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 16 ; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]] ; AUTO_VEC: vector.ph: ; AUTO_VEC-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792 @@ -264,7 +264,7 @@ define double @external_use_without_fast_math(ptr %a, i64 %n) { ; AUTO_VEC-NEXT: entry: ; AUTO_VEC-NEXT: [[SMAX:%.*]] = tail call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) ; AUTO_VEC-NEXT: [[XTRAITER:%.*]] = and i64 [[SMAX]], 7 -; AUTO_VEC-NEXT: [[TMP0:%.*]] = icmp ult i64 [[SMAX]], 8 +; AUTO_VEC-NEXT: [[TMP0:%.*]] = icmp slt i64 [[N]], 8 ; AUTO_VEC-NEXT: br i1 [[TMP0]], label [[FOR_END_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] ; AUTO_VEC: entry.new: ; AUTO_VEC-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[SMAX]], 9223372036854775800 diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll index 9b432de35c9d..8783326b1ef1 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll @@ -9,19 +9,18 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) { ; CHECK-NEXT: iter.check: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 8 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 -; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] -; CHECK-NEXT: [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 -; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[B]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[A]] +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[A]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] ; CHECK: vector.main.loop.iter.check: -; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[SMAX2]], 16 +; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp slt i64 [[N]], 16 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK3]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX2]], 9223372036854775792 @@ -33,13 +32,13 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) { ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP1]], align 4, !alias.scope !0, !noalias !3 +; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP1]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <16 x ptr> [[BROADCAST_SPLAT]], zeroinitializer -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> [[BROADCAST_SPLAT]], i32 4, <16 x i1> [[TMP3]], <16 x i32> poison), !alias.scope !3 +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> [[BROADCAST_SPLAT]], i32 4, <16 x i1> [[TMP3]], <16 x i32> poison), !alias.scope [[META3]] ; CHECK-NEXT: [[PREDPHI:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[WIDE_MASKED_GATHER]], <16 x i32> ; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[PREDPHI]], i64 15 ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC]] @@ -51,23 +50,23 @@ define i32 @inv_load_conditional(ptr %a, i64 %n, ptr %b, i32 %k) { ; CHECK: vec.epilog.ph: ; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; CHECK-NEXT: [[N_VEC7:%.*]] = and i64 [[SMAX2]], 9223372036854775800 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x ptr> poison, ptr [[A]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x ptr> [[BROADCAST_SPLATINSERT11]], <8 x ptr> poison, <8 x i32> zeroinitializer -; CHECK-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <8 x i32> poison, i32 [[NTRUNC]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT13]], <8 x i32> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <8 x ptr> poison, ptr [[A]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <8 x ptr> [[BROADCAST_SPLATINSERT10]], <8 x ptr> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <8 x i32> poison, i32 [[NTRUNC]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT12]], <8 x i32> poison, <8 x i32> zeroinitializer ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: -; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] +; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT16:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX9]] -; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT14]], ptr [[TMP5]], align 4, !alias.scope !8, !noalias !11 -; CHECK-NEXT: [[INDEX_NEXT17]] = add nuw i64 [[INDEX9]], 8 -; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC7]] +; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT13]], ptr [[TMP5]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]] +; CHECK-NEXT: [[INDEX_NEXT16]] = add nuw i64 [[INDEX9]], 8 +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT16]], [[N_VEC7]] ; CHECK-NEXT: br i1 [[TMP6]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] ; CHECK: vec.epilog.middle.block: -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <8 x ptr> [[BROADCAST_SPLAT12]], zeroinitializer -; CHECK-NEXT: [[WIDE_MASKED_GATHER15:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[BROADCAST_SPLAT12]], i32 4, <8 x i1> [[TMP7]], <8 x i32> poison), !alias.scope !11 -; CHECK-NEXT: [[PREDPHI16:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> [[WIDE_MASKED_GATHER15]], <8 x i32> -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[PREDPHI16]], i64 7 +; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <8 x ptr> [[BROADCAST_SPLAT11]], zeroinitializer +; CHECK-NEXT: [[WIDE_MASKED_GATHER14:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[BROADCAST_SPLAT11]], i32 4, <8 x i1> [[TMP7]], <8 x i32> poison), !alias.scope [[META11]] +; CHECK-NEXT: [[PREDPHI15:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> [[WIDE_MASKED_GATHER14]], <8 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[PREDPHI15]], i64 7 ; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[SMAX2]], [[N_VEC7]] ; CHECK-NEXT: br i1 [[CMP_N8]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]] ; CHECK: vec.epilog.scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll index 292ab4e4b2c4..330acb008d0d 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll @@ -12,19 +12,18 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b) ; CHECK-NEXT: iter.check: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 8 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[A]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[B]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] ; CHECK: vector.main.loop.iter.check: -; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[SMAX2]], 64 +; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp slt i64 [[N]], 64 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK3]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX2]], 9223372036854775744 @@ -39,15 +38,15 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b) ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 16 ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 32 ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 48 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 8, !alias.scope !0 -; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i32>, ptr [[TMP2]], align 8, !alias.scope !0 -; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i32>, ptr [[TMP3]], align 8, !alias.scope !0 -; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i32>, ptr [[TMP4]], align 8, !alias.scope !0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META0:![0-9]+]] +; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i32>, ptr [[TMP2]], align 8, !alias.scope [[META0]] +; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i32>, ptr [[TMP3]], align 8, !alias.scope [[META0]] +; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i32>, ptr [[TMP4]], align 8, !alias.scope [[META0]] ; CHECK-NEXT: [[TMP5]] = add <16 x i32> [[VEC_PHI]], [[WIDE_LOAD]] ; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[VEC_PHI4]], [[WIDE_LOAD7]] ; CHECK-NEXT: [[TMP7]] = add <16 x i32> [[VEC_PHI5]], [[WIDE_LOAD8]] ; CHECK-NEXT: [[TMP8]] = add <16 x i32> [[VEC_PHI6]], [[WIDE_LOAD9]] -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !3, !noalias !0 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -72,9 +71,9 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b) ; CHECK-NEXT: [[INDEX15:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT18:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI16:%.*]] = phi <8 x i32> [ [[TMP11]], [[VEC_EPILOG_PH]] ], [ [[TMP13:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX15]] -; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i32>, ptr [[TMP12]], align 8, !alias.scope !8 +; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i32>, ptr [[TMP12]], align 8, !alias.scope [[META8:![0-9]+]] ; CHECK-NEXT: [[TMP13]] = add <8 x i32> [[VEC_PHI16]], [[WIDE_LOAD17]] -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !11, !noalias !8 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META8]] ; CHECK-NEXT: [[INDEX_NEXT18]] = add nuw i64 [[INDEX15]], 8 ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT18]], [[N_VEC13]] ; CHECK-NEXT: br i1 [[TMP14]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] @@ -127,11 +126,10 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3 ; CHECK-NEXT: iter.check: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 8 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]] @@ -139,7 +137,7 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3 ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] ; CHECK: vector.main.loop.iter.check: -; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp ult i64 [[SMAX2]], 16 +; CHECK-NEXT: [[MIN_ITERS_CHECK3:%.*]] = icmp slt i64 [[N]], 16 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK3]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX2]], 9223372036854775792 @@ -153,10 +151,10 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3 ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 8, !alias.scope !15, !noalias !18 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META15:![0-9]+]], !noalias [[META18:![0-9]+]] ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP1]], align 4, !alias.scope !15, !noalias !18 -; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> [[BROADCAST_SPLAT5]], <16 x ptr> [[BROADCAST_SPLAT7]], i32 4, <16 x i1> [[TMP2]]), !alias.scope !18 +; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP1]], align 4, !alias.scope [[META15]], !noalias [[META18]] +; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> [[BROADCAST_SPLAT5]], <16 x ptr> [[BROADCAST_SPLAT7]], i32 4, <16 x i1> [[TMP2]]), !alias.scope [[META18]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] @@ -180,10 +178,10 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3 ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX11:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX11]] -; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i32>, ptr [[TMP4]], align 8, !alias.scope !21, !noalias !24 +; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i32>, ptr [[TMP4]], align 8, !alias.scope [[META21:![0-9]+]], !noalias [[META24:![0-9]+]] ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD12]], [[BROADCAST_SPLAT14]] -; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT16]], ptr [[TMP4]], align 4, !alias.scope !21, !noalias !24 -; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[BROADCAST_SPLAT16]], <8 x ptr> [[BROADCAST_SPLAT18]], i32 4, <8 x i1> [[TMP5]]), !alias.scope !24 +; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT16]], ptr [[TMP4]], align 4, !alias.scope [[META21]], !noalias [[META24]] +; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[BROADCAST_SPLAT16]], <8 x ptr> [[BROADCAST_SPLAT18]], i32 4, <8 x i1> [[TMP5]]), !alias.scope [[META24]] ; CHECK-NEXT: [[INDEX_NEXT19]] = add nuw i64 [[INDEX11]], 8 ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT19]], [[N_VEC9]] ; CHECK-NEXT: br i1 [[TMP6]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] @@ -240,11 +238,10 @@ define void @variant_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b ; CHECK-NEXT: iter.check: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX10:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX10]], 8 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[TMP0]] @@ -261,7 +258,7 @@ define void @variant_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b ; CHECK-NEXT: [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]] ; CHECK-NEXT: br i1 [[CONFLICT_RDX9]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] ; CHECK: vector.main.loop.iter.check: -; CHECK-NEXT: [[MIN_ITERS_CHECK11:%.*]] = icmp ult i64 [[SMAX10]], 16 +; CHECK-NEXT: [[MIN_ITERS_CHECK11:%.*]] = icmp slt i64 [[N]], 16 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK11]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX10]], 9223372036854775792 @@ -275,12 +272,12 @@ define void @variant_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 8, !alias.scope !28, !noalias !31 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META28:![0-9]+]], !noalias [[META31:![0-9]+]] ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT13]], ptr [[TMP1]], align 4, !alias.scope !28, !noalias !31 +; CHECK-NEXT: store <16 x i32> [[BROADCAST_SPLAT13]], ptr [[TMP1]], align 4, !alias.scope [[META28]], !noalias [[META31]] ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[C]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP3]], i32 8, <16 x i1> [[TMP2]], <16 x i32> poison), !alias.scope !34 -; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> [[WIDE_MASKED_LOAD]], <16 x ptr> [[BROADCAST_SPLAT15]], i32 4, <16 x i1> [[TMP2]]), !alias.scope !35, !noalias !34 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP3]], i32 8, <16 x i1> [[TMP2]], <16 x i32> poison), !alias.scope [[META34:![0-9]+]] +; CHECK-NEXT: call void @llvm.masked.scatter.v16i32.v16p0(<16 x i32> [[WIDE_MASKED_LOAD]], <16 x ptr> [[BROADCAST_SPLAT15]], i32 4, <16 x i1> [[TMP2]]), !alias.scope [[META35:![0-9]+]], !noalias [[META34]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] @@ -304,12 +301,12 @@ define void @variant_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX19:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT28:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX19]] -; CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i32>, ptr [[TMP5]], align 8, !alias.scope !37, !noalias !40 +; CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i32>, ptr [[TMP5]], align 8, !alias.scope [[META37:![0-9]+]], !noalias [[META40:![0-9]+]] ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD20]], [[BROADCAST_SPLAT22]] -; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT24]], ptr [[TMP5]], align 4, !alias.scope !37, !noalias !40 +; CHECK-NEXT: store <8 x i32> [[BROADCAST_SPLAT24]], ptr [[TMP5]], align 4, !alias.scope [[META37]], !noalias [[META40]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[C]], i64 [[INDEX19]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD25:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr [[TMP7]], i32 8, <8 x i1> [[TMP6]], <8 x i32> poison), !alias.scope !43 -; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[WIDE_MASKED_LOAD25]], <8 x ptr> [[BROADCAST_SPLAT27]], i32 4, <8 x i1> [[TMP6]]), !alias.scope !44, !noalias !43 +; CHECK-NEXT: [[WIDE_MASKED_LOAD25:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr [[TMP7]], i32 8, <8 x i1> [[TMP6]], <8 x i32> poison), !alias.scope [[META43:![0-9]+]] +; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[WIDE_MASKED_LOAD25]], <8 x ptr> [[BROADCAST_SPLAT27]], i32 4, <8 x i1> [[TMP6]]), !alias.scope [[META44:![0-9]+]], !noalias [[META43]] ; CHECK-NEXT: [[INDEX_NEXT28]] = add nuw i64 [[INDEX19]], 8 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT28]], [[N_VEC17]] ; CHECK-NEXT: br i1 [[TMP8]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll index 24c52b704952..736b0598043e 100644 --- a/llvm/test/Transforms/LoopVectorize/float-induction.ll +++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll @@ -1315,7 +1315,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) { ; VEC4_INTERL1-LABEL: @non_primary_iv_float_scalar( ; VEC4_INTERL1-NEXT: entry: ; VEC4_INTERL1-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 +; VEC4_INTERL1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; VEC4_INTERL1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; VEC4_INTERL1: vector.ph: ; VEC4_INTERL1-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 @@ -1392,7 +1392,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) { ; VEC4_INTERL2-LABEL: @non_primary_iv_float_scalar( ; VEC4_INTERL2-NEXT: entry: ; VEC4_INTERL2-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8 +; VEC4_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 ; VEC4_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; VEC4_INTERL2: vector.ph: ; VEC4_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775800 @@ -1508,7 +1508,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) { ; VEC1_INTERL2-LABEL: @non_primary_iv_float_scalar( ; VEC1_INTERL2-NEXT: entry: ; VEC1_INTERL2-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 +; VEC1_INTERL2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 ; VEC1_INTERL2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; VEC1_INTERL2: vector.ph: ; VEC1_INTERL2-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 @@ -1566,7 +1566,7 @@ define void @non_primary_iv_float_scalar(ptr %A, i64 %N) { ; VEC2_INTERL1_PRED_STORE-LABEL: @non_primary_iv_float_scalar( ; VEC2_INTERL1_PRED_STORE-NEXT: entry: ; VEC2_INTERL1_PRED_STORE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 +; VEC2_INTERL1_PRED_STORE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 ; VEC2_INTERL1_PRED_STORE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]] ; VEC2_INTERL1_PRED_STORE: vector.ph: ; VEC2_INTERL1_PRED_STORE-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll index d73193392e39..119c0aefaa73 100644 --- a/llvm/test/Transforms/LoopVectorize/induction.ll +++ b/llvm/test/Transforms/LoopVectorize/induction.ll @@ -670,7 +670,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) { ; IND-LABEL: @scalarize_induction_variable_01( ; IND-NEXT: entry: ; IND-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 +; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 ; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; IND: vector.ph: ; IND-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 @@ -708,7 +708,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) { ; UNROLL-LABEL: @scalarize_induction_variable_01( ; UNROLL-NEXT: entry: ; UNROLL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 +; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; UNROLL: vector.ph: ; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 @@ -799,7 +799,7 @@ define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) { ; INTERLEAVE-LABEL: @scalarize_induction_variable_01( ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8 +; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; INTERLEAVE: vector.ph: ; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775800 @@ -937,7 +937,7 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) { ; IND-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 ; IND-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 ; IND-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 -; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 9 +; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 9 ; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; IND: vector.ph: ; IND-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 4611686018427387902 @@ -997,7 +997,7 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) { ; UNROLL-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 ; UNROLL-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 ; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 -; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 25 +; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 25 ; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; UNROLL: vector.ph: ; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 4611686018427387900 @@ -1153,11 +1153,10 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) { ; ; INTERLEAVE-LABEL: @scalarize_induction_variable_02( ; INTERLEAVE-NEXT: entry: -; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 8) -; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 65 +; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 65 ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; INTERLEAVE: vector.ph: -; INTERLEAVE-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; INTERLEAVE-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -1 ; INTERLEAVE-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 ; INTERLEAVE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 ; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 7 @@ -1298,7 +1297,7 @@ define void @scalarize_induction_variable_03(ptr %p, i32 %y, i64 %n) { ; IND-LABEL: @scalarize_induction_variable_03( ; IND-NEXT: entry: ; IND-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 +; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 ; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; IND: vector.ph: ; IND-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 @@ -1343,7 +1342,7 @@ define void @scalarize_induction_variable_03(ptr %p, i32 %y, i64 %n) { ; UNROLL-LABEL: @scalarize_induction_variable_03( ; UNROLL-NEXT: entry: ; UNROLL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 +; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; UNROLL: vector.ph: ; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 @@ -1460,14 +1459,13 @@ define void @scalarize_induction_variable_03(ptr %p, i32 %y, i64 %n) { ; ; INTERLEAVE-LABEL: @scalarize_induction_variable_03( ; INTERLEAVE-NEXT: entry: -; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 9 +; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 9 ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; INTERLEAVE: vector.ph: -; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 7 +; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7 ; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; INTERLEAVE-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 8, i64 [[N_MOD_VF]] -; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 0 ; INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer ; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -2036,7 +2034,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) { ; IND-LABEL: @scalarize_induction_variable_05( ; IND-NEXT: entry: ; IND-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) -; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SMAX]], 2 +; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i32 [[N]], 2 ; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; IND: vector.ph: ; IND-NEXT: [[N_VEC:%.*]] = and i32 [[SMAX]], 2147483646 @@ -2103,7 +2101,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) { ; UNROLL-LABEL: @scalarize_induction_variable_05( ; UNROLL-NEXT: entry: ; UNROLL-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) -; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SMAX]], 4 +; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i32 [[N]], 4 ; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; UNROLL: vector.ph: ; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[SMAX]], 2147483644 @@ -2291,7 +2289,7 @@ define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) { ; INTERLEAVE-LABEL: @scalarize_induction_variable_05( ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) -; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SMAX]], 8 +; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i32 [[N]], 8 ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; INTERLEAVE: vector.ph: ; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i32 [[SMAX]], 2147483640 @@ -4921,7 +4919,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) { ; IND-LABEL: @non_primary_iv_trunc( ; IND-NEXT: entry: ; IND-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 +; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 ; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; IND: vector.ph: ; IND-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 @@ -4959,7 +4957,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) { ; UNROLL-LABEL: @non_primary_iv_trunc( ; UNROLL-NEXT: entry: ; UNROLL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 +; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; UNROLL: vector.ph: ; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 @@ -5046,7 +5044,7 @@ define void @non_primary_iv_trunc(ptr %a, i64 %n) { ; INTERLEAVE-LABEL: @non_primary_iv_trunc( ; INTERLEAVE-NEXT: entry: ; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8 +; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; INTERLEAVE: vector.ph: ; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775800 diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll index 7e647b883752..86ca1222aa4e 100644 --- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll +++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll @@ -17,14 +17,13 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" define void @interleaved_with_cond_store_0(ptr %p, i64 %x, i64 %n) { ; CHECK-LABEL: @interleaved_with_cond_store_0( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 3 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 3 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 1 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 2, i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -33,16 +32,16 @@ define void @interleaved_with_cond_store_0(ptr %p, i64 %x, i64 %n) { ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0 -; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; CHECK: pred.store.if: -; CHECK-NEXT: [[TMP2_1:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1 ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0 -; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP2_1]], align 8 +; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP5]], align 8 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] ; CHECK: pred.store.continue: -; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1 +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1 ; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]] ; CHECK: pred.store.if1: ; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 @@ -71,7 +70,7 @@ define void @interleaved_with_cond_store_0(ptr %p, i64 %x, i64 %n) { ; CHECK: if.merge: ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: for.end: ; CHECK-NEXT: ret void ; @@ -112,14 +111,13 @@ for.end: define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) { ; CHECK-LABEL: @interleaved_with_cond_store_1( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 3 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 3 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 1 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 2, i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -131,16 +129,16 @@ define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) { ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP2]], i32 1 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8 ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; CHECK: pred.store.if: -; CHECK-NEXT: [[PTR0:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 0 ; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0 -; CHECK-NEXT: store i64 [[TMP9]], ptr [[PTR0]], align 8 +; CHECK-NEXT: store i64 [[TMP9]], ptr [[TMP8]], align 8 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] ; CHECK: pred.store.continue: -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1 +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP6]], i64 1 ; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]] ; CHECK: pred.store.if1: ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP2]], i32 0 @@ -149,13 +147,13 @@ define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) { ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]] ; CHECK: pred.store.continue2: ; CHECK-NEXT: [[WIDE_VEC3:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8 -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 0 -; CHECK-NEXT: store i64 [[TMP14]], ptr [[TMP4]], align 8 -; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 2 -; CHECK-NEXT: store i64 [[TMP15]], ptr [[TMP5]], align 8 +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 0 +; CHECK-NEXT: store i64 [[TMP13]], ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 2 +; CHECK-NEXT: store i64 [[TMP14]], ptr [[TMP5]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: br label [[SCALAR_PH]] ; CHECK: scalar.ph: @@ -165,15 +163,15 @@ define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) { ; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[IF_MERGE:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] ; CHECK-NEXT: [[P_0:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0 ; CHECK-NEXT: [[P_1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1 -; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[P_1]], align 8 -; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], [[X]] -; CHECK-NEXT: br i1 [[TMP18]], label [[IF_THEN:%.*]], label [[IF_MERGE]] +; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[P_1]], align 8 +; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], [[X]] +; CHECK-NEXT: br i1 [[TMP17]], label [[IF_THEN:%.*]], label [[IF_MERGE]] ; CHECK: if.then: -; CHECK-NEXT: store i64 [[TMP17]], ptr [[P_0]], align 8 +; CHECK-NEXT: store i64 [[TMP16]], ptr [[P_0]], align 8 ; CHECK-NEXT: br label [[IF_MERGE]] ; CHECK: if.merge: -; CHECK-NEXT: [[TMP19:%.*]] = load i64, ptr [[P_0]], align 8 -; CHECK-NEXT: store i64 [[TMP19]], ptr [[P_1]], align 8 +; CHECK-NEXT: [[TMP18:%.*]] = load i64, ptr [[P_0]], align 8 +; CHECK-NEXT: store i64 [[TMP18]], ptr [[P_1]], align 8 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] ; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP5:![0-9]+]] @@ -219,14 +217,13 @@ for.end: define void @interleaved_with_cond_store_2(ptr %p, i64 %x, i64 %n) { ; CHECK-LABEL: @interleaved_with_cond_store_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 3 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 3 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 1 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 2, i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -240,16 +237,16 @@ define void @interleaved_with_cond_store_2(ptr %p, i64 %x, i64 %n) { ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> ; CHECK-NEXT: store i64 [[X]], ptr [[TMP3]], align 8 ; CHECK-NEXT: store i64 [[X]], ptr [[TMP4]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; CHECK: pred.store.if: -; CHECK-NEXT: [[PTR1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1 ; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0 -; CHECK-NEXT: store i64 [[TMP9]], ptr [[PTR1]], align 8 +; CHECK-NEXT: store i64 [[TMP9]], ptr [[TMP8]], align 8 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] ; CHECK: pred.store.continue: -; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1 +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP6]], i64 1 ; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]] ; CHECK: pred.store.if1: ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP2]], i32 1 diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll index 3b9edb11c5da..9a27c294f138 100644 --- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll @@ -896,14 +896,13 @@ for.body: ; preds = %for.body, %entry define void @PR27626_0(ptr %p, i32 %z, i64 %n) { ; CHECK-LABEL: @PR27626_0( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 5 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -984,14 +983,13 @@ for.end: define i32 @PR27626_1(ptr %p, i64 %n) { ; CHECK-LABEL: @PR27626_1( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 5 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1075,14 +1073,13 @@ for.end: define void @PR27626_2(ptr %p, i64 %n, i32 %z) { ; CHECK-LABEL: @PR27626_2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 5 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1167,14 +1164,13 @@ for.end: define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) { ; CHECK-LABEL: @PR27626_3( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 5 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 5 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 4, i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP1]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1274,7 +1270,7 @@ define void @PR27626_4(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) { ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 7 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 7 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 9223372036854775804 diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll index 1652a8e4ca37..50c67040cfb2 100644 --- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll +++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll @@ -22,11 +22,10 @@ define void @inv_val_store_to_inv_address_conditional_diff_values_ic(ptr %a, i64 ; CHECK-NEXT: entry: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]] @@ -123,11 +122,10 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[NTRUNC]], [[K:%.*]] ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]] @@ -213,12 +211,11 @@ define i32 @variant_val_store_to_inv_address(ptr %a, i64 %n, ptr %b, i32 %k) { ; CHECK-LABEL: @variant_val_store_to_inv_address( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[A]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[B]] diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll index c4d7ef8e949b..20d612a548b1 100644 --- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll @@ -21,12 +21,11 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b) ; CHECK-NEXT: entry: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[A]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[B]] @@ -39,9 +38,9 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b) ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope !0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META0:![0-9]+]] ; CHECK-NEXT: [[TMP2]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]] -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !3, !noalias !0 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META0]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -96,12 +95,11 @@ define void @inv_val_store_to_inv_address(ptr %a, i64 %n, ptr %b) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[A]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[B]] @@ -115,8 +113,8 @@ define void @inv_val_store_to_inv_address(ptr %a, i64 %n, ptr %b) { ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !9, !noalias !12 -; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4, !alias.scope !12 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META12:![0-9]+]] +; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4, !alias.scope [[META12]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -172,11 +170,10 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3 ; CHECK-NEXT: entry: ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 ; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: -; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SMAX]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]] ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]] @@ -193,31 +190,31 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3 ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE10:%.*]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope !16, !noalias !19 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 8, !alias.scope [[META16:![0-9]+]], !noalias [[META19:![0-9]+]] ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT4]], ptr [[TMP1]], align 4, !alias.scope !16, !noalias !19 +; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT4]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META19]] ; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 ; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] ; CHECK: pred.store.if: -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !19 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META19]] ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] ; CHECK: pred.store.continue: ; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 ; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]] ; CHECK: pred.store.if5: -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !19 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META19]] ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]] ; CHECK: pred.store.continue6: ; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 ; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]] ; CHECK: pred.store.if7: -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !19 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META19]] ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]] ; CHECK: pred.store.continue8: ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 ; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10]] ; CHECK: pred.store.if9: -; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope !19 +; CHECK-NEXT: store i32 [[NTRUNC]], ptr [[A]], align 4, !alias.scope [[META19]] ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]] ; CHECK: pred.store.continue10: ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 @@ -391,7 +388,7 @@ define i32 @multiple_uniform_stores(ptr nocapture %var1, ptr nocapture readonly ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP15]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[INVARIANT_GEP]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[GEP]], align 4, !alias.scope !23 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[GEP]], align 4, !alias.scope [[META23:![0-9]+]] ; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]] ; CHECK-NEXT: [[TMP17]] = add <4 x i32> [[TMP16]], ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 diff --git a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll index 24cbf7d0e386..de298d20fc38 100644 --- a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll +++ b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll @@ -9,7 +9,7 @@ define void @vector_gep(ptr %a, ptr %b, i64 %n) { ; CHECK-LABEL: @vector_gep( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 @@ -65,7 +65,7 @@ define void @scalar_store(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 3 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 3 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 9223372036854775806 @@ -125,7 +125,7 @@ define void @expansion(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 3 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 3 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 9223372036854775806 @@ -182,7 +182,7 @@ define void @no_gep_or_bitcast(ptr noalias %a, i64 %n) { ; CHECK-LABEL: @no_gep_or_bitcast( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 diff --git a/llvm/test/Transforms/LoopVectorize/reduction.ll b/llvm/test/Transforms/LoopVectorize/reduction.ll index 7c12eb1d4e59..757b41db6a5b 100644 --- a/llvm/test/Transforms/LoopVectorize/reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/reduction.ll @@ -1201,7 +1201,7 @@ define i64 @reduction_with_phi_with_one_incoming_on_backedge(i16 %n, ptr %A) { ; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 2) ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i16 [[SMAX]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i16 [[TMP0]] to i32 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i16 [[SMAX]], 5 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i16 [[N]], 5 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP1]], 32764 @@ -1279,7 +1279,7 @@ define i64 @reduction_with_phi_with_two_incoming_on_backedge(i16 %n, ptr %A) { ; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 2) ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i16 [[SMAX]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i16 [[TMP0]] to i32 -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i16 [[SMAX]], 5 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i16 [[N]], 5 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP1]], 32764 diff --git a/llvm/test/Transforms/LoopVectorize/vector-geps.ll b/llvm/test/Transforms/LoopVectorize/vector-geps.ll index 4f4b4f6600da..ec0b4864ac3f 100644 --- a/llvm/test/Transforms/LoopVectorize/vector-geps.ll +++ b/llvm/test/Transforms/LoopVectorize/vector-geps.ll @@ -8,7 +8,7 @@ define void @vector_gep_stored(ptr %a, ptr %b, i64 %n) { ; CHECK-LABEL: @vector_gep_stored( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 @@ -21,8 +21,8 @@ define void @vector_gep_stored(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: store <4 x ptr> [[TMP0]], ptr [[TMP1]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], -; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] @@ -36,7 +36,7 @@ define void @vector_gep_stored(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: store ptr [[VAR0]], ptr [[VAR1]], align 8 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: for.end: ; CHECK-NEXT: ret void ; @@ -61,7 +61,7 @@ define void @uniform_vector_gep_stored(ptr %a, ptr %b, i64 %n) { ; CHECK-LABEL: @uniform_vector_gep_stored( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 @@ -74,8 +74,8 @@ define void @uniform_vector_gep_stored(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds ptr, ptr [[A:%.*]], i64 [[INDEX]] ; CHECK-NEXT: store <4 x ptr> [[DOTSPLAT]], ptr [[TMP1]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -- GitLab From e0c554ad87d18dcbfcb9b6485d0da800ae1338d1 Mon Sep 17 00:00:00 2001 From: Nick Anderson Date: Thu, 4 Jan 2024 22:47:56 -0800 Subject: [PATCH 071/728] =?UTF-8?q?Port=20CodeGenPrepare=20to=20new=20pass?= =?UTF-8?q?=20manager=20(and=20BasicBlockSectionsProfil=E2=80=A6=20(#75380?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port CodeGenPrepare to new pass manager and dependency BasicBlockSectionsProfileReader Fixes: #64560 Co-authored-by: Krishna-13-cyber <84722531+Krishna-13-cyber@users.noreply.github.com> --- .../CodeGen/BasicBlockSectionsProfileReader.h | 83 +++++++++--- llvm/include/llvm/CodeGen/CodeGenPrepare.h | 35 +++++ llvm/include/llvm/CodeGen/Passes.h | 4 +- llvm/include/llvm/InitializePasses.h | 4 +- llvm/include/llvm/LinkAllPasses.h | 2 +- llvm/lib/CodeGen/BasicBlockPathCloning.cpp | 11 +- llvm/lib/CodeGen/BasicBlockSections.cpp | 8 +- .../BasicBlockSectionsProfileReader.cpp | 52 ++++++-- llvm/lib/CodeGen/CodeGen.cpp | 2 +- llvm/lib/CodeGen/CodeGenPrepare.cpp | 124 ++++++++++++------ llvm/lib/CodeGen/TargetPassConfig.cpp | 4 +- llvm/lib/Passes/PassBuilder.cpp | 2 + llvm/lib/Passes/PassRegistry.def | 2 + .../AArch64/aarch64-codegen-prepare-atp.ll | 2 +- llvm/test/CodeGen/AArch64/and-sink.ll | 4 +- .../CodeGen/AArch64/arm64-bitfield-extract.ll | 2 +- .../AArch64/arm64-codegen-prepare-extload.ll | 6 +- .../test/CodeGen/AArch64/arm64_32-gep-sink.ll | 2 +- .../CodeGen/AArch64/cgp-trivial-phi-node.ll | 2 +- llvm/test/CodeGen/AArch64/convertphitype.ll | 2 +- .../AArch64/scalable-vector-promotion.ll | 2 +- llvm/test/CodeGen/AArch64/sve-vscale.ll | 2 +- .../CodeGen/AArch64/sve2-vscale-sinking.ll | 2 +- .../AMDGPU/cgp-addressing-modes-flat.ll | 8 +- .../AMDGPU/cgp-addressing-modes-gfx1030.ll | 2 +- .../AMDGPU/cgp-addressing-modes-gfx908.ll | 2 +- .../CodeGen/AMDGPU/cgp-addressing-modes.ll | 8 +- llvm/test/CodeGen/ARM/vector-promotion.ll | 4 +- .../Generic/addr-sink-call-multi-arg.ll | 2 +- llvm/test/CodeGen/Generic/addr-use-count.ll | 2 +- .../test/CodeGen/X86/callbr-codegenprepare.ll | 2 +- .../X86/codegen-prepare-addrmode-sext.ll | 2 +- .../CodeGen/X86/codegen-prepare-extload.ll | 6 +- llvm/test/CodeGen/X86/convertphitype.ll | 2 +- .../CodeGen/X86/indirect-br-gep-unmerge.ll | 2 +- llvm/test/CodeGen/X86/pr58538.ll | 4 +- llvm/test/CodeGen/X86/tailcall-cgp-dup.ll | 2 +- llvm/test/CodeGen/X86/tailcall-extract.ll | 2 +- llvm/test/DebugInfo/ARM/salvage-debug-info.ll | 2 +- llvm/test/DebugInfo/X86/zextload.ll | 2 +- llvm/test/Other/codegenprepare-and-debug.ll | 2 +- .../AArch64/combine-address-mode.ll | 2 +- .../CodeGenPrepare/AArch64/free-zext.ll | 2 +- .../gather-scatter-opt-inseltpoison.ll | 2 +- .../AArch64/gather-scatter-opt.ll | 2 +- .../AArch64/overflow-intrinsics.ll | 6 +- .../AArch64/sink-gather-scatter-addressing.ll | 2 +- .../AArch64/trunc-weird-user.ll | 2 +- .../CodeGenPrepare/AArch64/zext-to-shuffle.ll | 2 +- .../CodeGenPrepare/AMDGPU/addressing-modes.ll | 2 +- .../AMDGPU/no-sink-addrspacecast.ll | 2 +- .../AMDGPU/sink-addrspacecast.ll | 2 +- .../CodeGenPrepare/ARM/branch-on-zero.ll | 2 +- .../Transforms/CodeGenPrepare/ARM/dead-gep.ll | 2 +- .../CodeGenPrepare/ARM/memory-intrinsics.ll | 2 +- .../CodeGenPrepare/ARM/overflow-intrinsics.ll | 2 +- .../CodeGenPrepare/ARM/sink-addrmode.ll | 2 +- .../Transforms/CodeGenPrepare/ARM/splitgep.ll | 2 +- .../CodeGenPrepare/ARM/tailcall-dup.ll | 2 +- .../bypass-slow-div-constant-numerator.ll | 2 +- .../NVPTX/bypass-slow-div-not-exact.ll | 2 +- .../NVPTX/bypass-slow-div-special-cases.ll | 2 +- .../CodeGenPrepare/NVPTX/bypass-slow-div.ll | 2 +- .../NVPTX/dont-introduce-addrspacecast.ll | 2 +- .../NVPTX/dont-sink-nop-addrspacecast.ll | 2 +- .../PowerPC/split-store-alignment.ll | 4 +- .../CodeGenPrepare/RISCV/and-mask-sink.ll | 8 +- .../CodeGenPrepare/RISCV/cttz-ctlz.ll | 2 +- .../SPARC/overflow-intrinsics.ll | 6 +- .../CodeGenPrepare/X86/catchpad-phi-cast.ll | 4 +- .../X86/cgp_shuffle_crash-inseltpoison.ll | 2 +- .../CodeGenPrepare/X86/cgp_shuffle_crash.ll | 2 +- .../CodeGenPrepare/X86/computedgoto.ll | 2 +- .../CodeGenPrepare/X86/cttz-ctlz.ll | 10 +- .../X86/delete-assume-dead-code.ll | 2 +- .../CodeGenPrepare/X86/extend-sink-hoist.ll | 2 +- .../CodeGenPrepare/X86/freeze-brcond.ll | 2 +- .../X86/gather-scatter-opt-inseltpoison.ll | 4 +- .../CodeGenPrepare/X86/gather-scatter-opt.ll | 2 +- .../CodeGenPrepare/X86/gep-unmerging.ll | 2 +- .../CodeGenPrepare/X86/invariant.group.ll | 2 +- .../X86/masked-gather-struct-gep.ll | 2 +- .../CodeGenPrepare/X86/nonintegral.ll | 4 +- .../CodeGenPrepare/X86/optimizeSelect-DT.ll | 2 +- .../CodeGenPrepare/X86/overflow-intrinsics.ll | 6 +- .../Transforms/CodeGenPrepare/X86/pr27536.ll | 2 +- .../Transforms/CodeGenPrepare/X86/pr35658.ll | 2 +- .../Transforms/CodeGenPrepare/X86/pr72046.ll | 2 +- .../recursively-delete-dead-instructions.ll | 2 +- .../CodeGenPrepare/X86/remove-assume-block.ll | 2 +- .../Transforms/CodeGenPrepare/X86/select.ll | 6 +- .../CodeGenPrepare/X86/sink-addrmode-base.ll | 4 +- .../X86/sink-addrmode-inseltpoison.ll | 2 +- .../X86/sink-addrmode-select.ll | 2 +- .../X86/sink-addrmode-two-phi.ll | 2 +- .../CodeGenPrepare/X86/sink-addrmode.ll | 2 +- .../CodeGenPrepare/X86/sink-addrspacecast.ll | 2 +- .../CodeGenPrepare/X86/split-indirect-loop.ll | 2 +- .../X86/split-store-alignment.ll | 2 +- .../CodeGenPrepare/X86/statepoint-relocate.ll | 2 +- .../CodeGenPrepare/X86/tailcall-assume-xbb.ll | 2 +- .../X86/vec-shift-inseltpoison.ll | 14 +- .../CodeGenPrepare/X86/vec-shift.ll | 14 +- .../CodeGenPrepare/X86/widenable-condition.ll | 2 +- .../X86/x86-shuffle-sink-inseltpoison.ll | 8 +- .../CodeGenPrepare/X86/x86-shuffle-sink.ll | 8 +- .../CodeGenPrepare/dead-allocation.ll | 2 +- .../CodeGenPrepare/skip-merging-case-block.ll | 2 +- .../Transforms/HotColdSplit/coldentrycount.ll | 2 +- .../codegenprepare-produced-address-math.ll | 2 +- .../section-accurate-samplepgo.ll | 6 +- llvm/tools/opt/opt.cpp | 2 +- 112 files changed, 399 insertions(+), 238 deletions(-) create mode 100644 llvm/include/llvm/CodeGen/CodeGenPrepare.h diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h index dfb8d5d9f2f5..bba675f1d3eb 100644 --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -21,11 +21,14 @@ #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Error.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Target/TargetMachine.h" + using namespace llvm; namespace llvm { @@ -72,25 +75,13 @@ template <> struct DenseMapInfo { } }; -class BasicBlockSectionsProfileReader : public ImmutablePass { +class BasicBlockSectionsProfileReader { public: - static char ID; - + friend class BasicBlockSectionsProfileReaderWrapperPass; BasicBlockSectionsProfileReader(const MemoryBuffer *Buf) - : ImmutablePass(ID), MBuf(Buf), - LineIt(*Buf, /*SkipBlanks=*/true, /*CommentMarker=*/'#') { - initializeBasicBlockSectionsProfileReaderPass( - *PassRegistry::getPassRegistry()); - }; + : MBuf(Buf), LineIt(*Buf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'){}; - BasicBlockSectionsProfileReader() : ImmutablePass(ID) { - initializeBasicBlockSectionsProfileReaderPass( - *PassRegistry::getPassRegistry()); - } - - StringRef getPassName() const override { - return "Basic Block Sections Profile Reader"; - } + BasicBlockSectionsProfileReader(){}; // Returns true if basic block sections profile exist for function \p // FuncName. @@ -109,10 +100,6 @@ public: SmallVector> getClonePathsForFunction(StringRef FuncName) const; - // Initializes the FunctionNameToDIFilename map for the current module and - // then reads the profile for the matching functions. - bool doInitialization(Module &M) override; - private: StringRef getAliasName(StringRef FuncName) const { auto R = FuncAliasMap.find(FuncName); @@ -170,7 +157,61 @@ private: // sections profile. \p Buf is a memory buffer that contains the list of // functions and basic block ids to selectively enable basic block sections. ImmutablePass * -createBasicBlockSectionsProfileReaderPass(const MemoryBuffer *Buf); +createBasicBlockSectionsProfileReaderWrapperPass(const MemoryBuffer *Buf); + +/// Analysis pass providing the \c BasicBlockSectionsProfileReader. +/// +/// Note that this pass's result cannot be invalidated, it is immutable for the +/// life of the module. +class BasicBlockSectionsProfileReaderAnalysis + : public AnalysisInfoMixin { + +public: + static AnalysisKey Key; + typedef BasicBlockSectionsProfileReader Result; + BasicBlockSectionsProfileReaderAnalysis(const TargetMachine *TM) : TM(TM) {} + + Result run(Function &F, FunctionAnalysisManager &AM); + +private: + const TargetMachine *TM; +}; + +class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass { +public: + static char ID; + BasicBlockSectionsProfileReader BBSPR; + + BasicBlockSectionsProfileReaderWrapperPass(const MemoryBuffer *Buf) + : ImmutablePass(ID), BBSPR(BasicBlockSectionsProfileReader(Buf)) { + initializeBasicBlockSectionsProfileReaderWrapperPassPass( + *PassRegistry::getPassRegistry()); + }; + + BasicBlockSectionsProfileReaderWrapperPass() + : ImmutablePass(ID), BBSPR(BasicBlockSectionsProfileReader()) { + initializeBasicBlockSectionsProfileReaderWrapperPassPass( + *PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { + return "Basic Block Sections Profile Reader"; + } + + bool isFunctionHot(StringRef FuncName) const; + + std::pair> + getClusterInfoForFunction(StringRef FuncName) const; + + SmallVector> + getClonePathsForFunction(StringRef FuncName) const; + + // Initializes the FunctionNameToDIFilename map for the current module and + // then reads the profile for the matching functions. + bool doInitialization(Module &M) override; + + BasicBlockSectionsProfileReader &getBBSPR(); +}; } // namespace llvm #endif // LLVM_CODEGEN_BASICBLOCKSECTIONSPROFILEREADER_H diff --git a/llvm/include/llvm/CodeGen/CodeGenPrepare.h b/llvm/include/llvm/CodeGen/CodeGenPrepare.h new file mode 100644 index 000000000000..dee3a9ee53d7 --- /dev/null +++ b/llvm/include/llvm/CodeGen/CodeGenPrepare.h @@ -0,0 +1,35 @@ +//===- CodeGenPrepare.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// Defines an IR pass for CodeGen Prepare. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_PREPARE_H +#define LLVM_CODEGEN_PREPARE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Function; +class TargetMachine; + +class CodeGenPreparePass : public PassInfoMixin { +private: + const TargetMachine *TM; + +public: + CodeGenPreparePass(const TargetMachine *TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_CODEGEN_PREPARE_H diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index ca9fbb1def76..bbfb8a0dbe26 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -93,9 +93,9 @@ namespace llvm { MachineFunctionPass *createResetMachineFunctionPass(bool EmitFallbackDiag, bool AbortOnFailedISel); - /// createCodeGenPreparePass - Transform the code to expose more pattern + /// createCodeGenPrepareLegacyPass - Transform the code to expose more pattern /// matching during instruction selection. - FunctionPass *createCodeGenPreparePass(); + FunctionPass *createCodeGenPrepareLegacyPass(); /// This pass implements generation of target-specific intrinsics to support /// handling of complex number arithmetic diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 46b1e95c3c15..3db639a68724 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -54,7 +54,7 @@ void initializeAssignmentTrackingAnalysisPass(PassRegistry &); void initializeAssumptionCacheTrackerPass(PassRegistry&); void initializeAtomicExpandPass(PassRegistry&); void initializeBasicBlockPathCloningPass(PassRegistry &); -void initializeBasicBlockSectionsProfileReaderPass(PassRegistry &); +void initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &); void initializeBasicBlockSectionsPass(PassRegistry &); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAAWrapperPassPass(PassRegistry&); @@ -75,7 +75,7 @@ void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); void initializeCallGraphWrapperPassPass(PassRegistry&); void initializeCheckDebugMachineModulePass(PassRegistry &); -void initializeCodeGenPreparePass(PassRegistry&); +void initializeCodeGenPrepareLegacyPassPass(PassRegistry &); void initializeComplexDeinterleavingLegacyPassPass(PassRegistry&); void initializeConstantHoistingLegacyPassPass(PassRegistry&); void initializeCycleInfoWrapperPassPass(PassRegistry &); diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index 7a21876e565a..fe7fedad18bc 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -113,7 +113,7 @@ namespace { (void) llvm::createTailCallEliminationPass(); (void)llvm::createTLSVariableHoistPass(); (void) llvm::createConstantHoistingPass(); - (void) llvm::createCodeGenPreparePass(); + (void)llvm::createCodeGenPrepareLegacyPass(); (void) llvm::createEarlyCSEPass(); (void) llvm::createGVNPass(); (void) llvm::createPostDomTree(); diff --git a/llvm/lib/CodeGen/BasicBlockPathCloning.cpp b/llvm/lib/CodeGen/BasicBlockPathCloning.cpp index 5d5f3c3da481..901542e8507b 100644 --- a/llvm/lib/CodeGen/BasicBlockPathCloning.cpp +++ b/llvm/lib/CodeGen/BasicBlockPathCloning.cpp @@ -196,7 +196,7 @@ class BasicBlockPathCloning : public MachineFunctionPass { public: static char ID; - BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr; + BasicBlockSectionsProfileReaderWrapperPass *BBSectionsProfileReader = nullptr; BasicBlockPathCloning() : MachineFunctionPass(ID) { initializeBasicBlockPathCloningPass(*PassRegistry::getPassRegistry()); @@ -218,7 +218,7 @@ INITIALIZE_PASS_BEGIN( BasicBlockPathCloning, "bb-path-cloning", "Applies path clonings for the -basic-block-sections=list option", false, false) -INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReader) +INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass) INITIALIZE_PASS_END( BasicBlockPathCloning, "bb-path-cloning", "Applies path clonings for the -basic-block-sections=list option", false, @@ -230,13 +230,14 @@ bool BasicBlockPathCloning::runOnMachineFunction(MachineFunction &MF) { if (hasInstrProfHashMismatch(MF)) return false; - return ApplyCloning(MF, getAnalysis() - .getClonePathsForFunction(MF.getName())); + return ApplyCloning(MF, + getAnalysis() + .getClonePathsForFunction(MF.getName())); } void BasicBlockPathCloning::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); - AU.addRequired(); + AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp index 42997d2287d6..94b5a503fbd0 100644 --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -103,7 +103,7 @@ class BasicBlockSections : public MachineFunctionPass { public: static char ID; - BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr; + BasicBlockSectionsProfileReaderWrapperPass *BBSectionsProfileReader = nullptr; BasicBlockSections() : MachineFunctionPass(ID) { initializeBasicBlockSectionsPass(*PassRegistry::getPassRegistry()); @@ -128,7 +128,7 @@ INITIALIZE_PASS_BEGIN( "Prepares for basic block sections, by splitting functions " "into clusters of basic blocks.", false, false) -INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReader) +INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass) INITIALIZE_PASS_END(BasicBlockSections, "bbsections-prepare", "Prepares for basic block sections, by splitting functions " "into clusters of basic blocks.", @@ -306,7 +306,7 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) { DenseMap FuncClusterInfo; if (BBSectionsType == BasicBlockSection::List) { auto [HasProfile, ClusterInfo] = - getAnalysis() + getAnalysis() .getClusterInfoForFunction(MF.getName()); if (!HasProfile) return false; @@ -362,7 +362,7 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) { void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); - AU.addRequired(); + AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp index 15b6f63e8632..79e42d9304df 100644 --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -30,8 +30,9 @@ using namespace llvm; -char BasicBlockSectionsProfileReader::ID = 0; -INITIALIZE_PASS(BasicBlockSectionsProfileReader, "bbsections-profile-reader", +char BasicBlockSectionsProfileReaderWrapperPass::ID = 0; +INITIALIZE_PASS(BasicBlockSectionsProfileReaderWrapperPass, + "bbsections-profile-reader", "Reads and parses a basic block sections profile.", false, false) @@ -395,11 +396,11 @@ Error BasicBlockSectionsProfileReader::ReadProfile() { } } -bool BasicBlockSectionsProfileReader::doInitialization(Module &M) { - if (!MBuf) +bool BasicBlockSectionsProfileReaderWrapperPass::doInitialization(Module &M) { + if (!BBSPR.MBuf) return false; // Get the function name to debug info filename mapping. - FunctionNameToDIFilename.clear(); + BBSPR.FunctionNameToDIFilename.clear(); for (const Function &F : M) { SmallString<128> DIFilename; if (F.isDeclaration()) @@ -411,15 +412,46 @@ bool BasicBlockSectionsProfileReader::doInitialization(Module &M) { DIFilename = sys::path::remove_leading_dotslash(CU->getFilename()); } [[maybe_unused]] bool inserted = - FunctionNameToDIFilename.try_emplace(F.getName(), DIFilename).second; + BBSPR.FunctionNameToDIFilename.try_emplace(F.getName(), DIFilename) + .second; assert(inserted); } - if (auto Err = ReadProfile()) + if (auto Err = BBSPR.ReadProfile()) report_fatal_error(std::move(Err)); return false; } -ImmutablePass * -llvm::createBasicBlockSectionsProfileReaderPass(const MemoryBuffer *Buf) { - return new BasicBlockSectionsProfileReader(Buf); +AnalysisKey BasicBlockSectionsProfileReaderAnalysis::Key; + +BasicBlockSectionsProfileReader +BasicBlockSectionsProfileReaderAnalysis::run(Function &F, + FunctionAnalysisManager &AM) { + return BasicBlockSectionsProfileReader(TM->getBBSectionsFuncListBuf()); +} + +bool BasicBlockSectionsProfileReaderWrapperPass::isFunctionHot( + StringRef FuncName) const { + return BBSPR.isFunctionHot(FuncName); +} + +std::pair> +BasicBlockSectionsProfileReaderWrapperPass::getClusterInfoForFunction( + StringRef FuncName) const { + return BBSPR.getClusterInfoForFunction(FuncName); +} + +SmallVector> +BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction( + StringRef FuncName) const { + return BBSPR.getClonePathsForFunction(FuncName); +} + +BasicBlockSectionsProfileReader & +BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() { + return BBSPR; +} + +ImmutablePass *llvm::createBasicBlockSectionsProfileReaderWrapperPass( + const MemoryBuffer *Buf) { + return new BasicBlockSectionsProfileReaderWrapperPass(Buf); } diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp index 7b73a7b11ddf..418066452c17 100644 --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -30,7 +30,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeCFIFixupPass(Registry); initializeCFIInstrInserterPass(Registry); initializeCheckDebugMachineModulePass(Registry); - initializeCodeGenPreparePass(Registry); + initializeCodeGenPrepareLegacyPassPass(Registry); initializeDeadMachineInstructionElimPass(Registry); initializeDebugifyMachineModulePass(Registry); initializeDetectDeadLanesPass(Registry); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 5bd4c6b067d7..35a70f0b8c23 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/CodeGenPrepare.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -301,7 +302,8 @@ using ValueToSExts = MapVector; class TypePromotionTransaction; -class CodeGenPrepare : public FunctionPass { +class CodeGenPrepare { + friend class CodeGenPrepareLegacyPass; const TargetMachine *TM = nullptr; const TargetSubtargetInfo *SubtargetInfo = nullptr; const TargetLowering *TLI = nullptr; @@ -365,6 +367,8 @@ class CodeGenPrepare : public FunctionPass { std::unique_ptr DT; public: + CodeGenPrepare(){}; + CodeGenPrepare(const TargetMachine *TM) : TM(TM){}; /// If encounter huge function, we need to limit the build time. bool IsHugeFunc = false; @@ -374,15 +378,7 @@ public: /// to insert such BB into FreshBBs for huge function. SmallSet FreshBBs; - static char ID; // Pass identification, replacement for typeid - - CodeGenPrepare() : FunctionPass(ID) { - initializeCodeGenPreparePass(*PassRegistry::getPassRegistry()); - } - - bool runOnFunction(Function &F) override; - - void releaseMemory() override { + void releaseMemory() { // Clear per function information. InsertedInsts.clear(); PromotedInsts.clear(); @@ -391,17 +387,7 @@ public: BFI.reset(); } - StringRef getPassName() const override { return "CodeGen Prepare"; } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - // FIXME: When we can selectively preserve passes, preserve the domtree. - AU.addRequired(); - AU.addRequired(); - AU.addRequired(); - AU.addRequired(); - AU.addRequired(); - AU.addUsedIfAvailable(); - } + bool run(Function &F, FunctionAnalysisManager &AM); private: template @@ -488,45 +474,107 @@ private: bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT); bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT); void verifyBFIUpdates(Function &F); + bool _run(Function &F); +}; + +class CodeGenPrepareLegacyPass : public FunctionPass { +public: + static char ID; // Pass identification, replacement for typeid + + CodeGenPrepareLegacyPass() : FunctionPass(ID) { + initializeCodeGenPrepareLegacyPassPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; + + StringRef getPassName() const override { return "CodeGen Prepare"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + // FIXME: When we can selectively preserve passes, preserve the domtree. + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addUsedIfAvailable(); + } }; } // end anonymous namespace -char CodeGenPrepare::ID = 0; +char CodeGenPrepareLegacyPass::ID = 0; -INITIALIZE_PASS_BEGIN(CodeGenPrepare, DEBUG_TYPE, +bool CodeGenPrepareLegacyPass::runOnFunction(Function &F) { + if (skipFunction(F)) + return false; + auto TM = &getAnalysis().getTM(); + CodeGenPrepare CGP(TM); + CGP.DL = &F.getParent()->getDataLayout(); + CGP.SubtargetInfo = TM->getSubtargetImpl(F); + CGP.TLI = CGP.SubtargetInfo->getTargetLowering(); + CGP.TRI = CGP.SubtargetInfo->getRegisterInfo(); + CGP.TLInfo = &getAnalysis().getTLI(F); + CGP.TTI = &getAnalysis().getTTI(F); + CGP.LI = &getAnalysis().getLoopInfo(); + CGP.BPI.reset(new BranchProbabilityInfo(F, *CGP.LI)); + CGP.BFI.reset(new BlockFrequencyInfo(F, *CGP.BPI, *CGP.LI)); + CGP.PSI = &getAnalysis().getPSI(); + auto BBSPRWP = + getAnalysisIfAvailable(); + CGP.BBSectionsProfileReader = BBSPRWP ? &BBSPRWP->getBBSPR() : nullptr; + + return CGP._run(F); +} + +INITIALIZE_PASS_BEGIN(CodeGenPrepareLegacyPass, DEBUG_TYPE, "Optimize for code generation", false, false) -INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReader) +INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass) INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) -INITIALIZE_PASS_END(CodeGenPrepare, DEBUG_TYPE, "Optimize for code generation", - false, false) +INITIALIZE_PASS_END(CodeGenPrepareLegacyPass, DEBUG_TYPE, + "Optimize for code generation", false, false) -FunctionPass *llvm::createCodeGenPreparePass() { return new CodeGenPrepare(); } +FunctionPass *llvm::createCodeGenPrepareLegacyPass() { + return new CodeGenPrepareLegacyPass(); +} -bool CodeGenPrepare::runOnFunction(Function &F) { - if (skipFunction(F)) - return false; +PreservedAnalyses CodeGenPreparePass::run(Function &F, + FunctionAnalysisManager &AM) { + CodeGenPrepare CGP(TM); - DL = &F.getParent()->getDataLayout(); + bool Changed = CGP.run(F, AM); + if (!Changed) + return PreservedAnalyses::all(); - bool EverMadeChange = false; + PreservedAnalyses PA; + PA.preserveSet(); + PA.preserve(); + return PA; +} - TM = &getAnalysis().getTM(); +bool CodeGenPrepare::run(Function &F, FunctionAnalysisManager &AM) { + DL = &F.getParent()->getDataLayout(); SubtargetInfo = TM->getSubtargetImpl(F); TLI = SubtargetInfo->getTargetLowering(); TRI = SubtargetInfo->getRegisterInfo(); - TLInfo = &getAnalysis().getTLI(F); - TTI = &getAnalysis().getTTI(F); - LI = &getAnalysis().getLoopInfo(); + TLInfo = &AM.getResult(F); + TTI = &AM.getResult(F); + LI = &AM.getResult(F); BPI.reset(new BranchProbabilityInfo(F, *LI)); BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); - PSI = &getAnalysis().getPSI(); + auto &MAMProxy = AM.getResult(F); + PSI = MAMProxy.getCachedResult(*F.getParent()); BBSectionsProfileReader = - getAnalysisIfAvailable(); + AM.getCachedResult(F); + return _run(F); +} + +bool CodeGenPrepare::_run(Function &F) { + bool EverMadeChange = false; + OptSize = F.hasOptSize(); // Use the basic-block-sections profile to promote hot functions to .text.hot // if requested. diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index 4003a08a5422..3bbc792f4cbf 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -978,7 +978,7 @@ void TargetPassConfig::addPassesToHandleExceptions() { /// before exception handling preparation passes. void TargetPassConfig::addCodeGenPrepare() { if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP) - addPass(createCodeGenPreparePass()); + addPass(createCodeGenPrepareLegacyPass()); } /// Add common passes that perform LLVM IR to IR transforms in preparation for @@ -1271,7 +1271,7 @@ void TargetPassConfig::addMachinePasses() { // together. Update this check once we have addressed any issues. if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) { if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) { - addPass(llvm::createBasicBlockSectionsProfileReaderPass( + addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass( TM->getBBSectionsFuncListBuf())); addPass(llvm::createBasicBlockPathCloningPass()); } diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 439f749bda8b..ca943b43404f 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -72,7 +72,9 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/Analysis/UniformityAnalysis.h" +#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h" #include "llvm/CodeGen/CallBrPrepare.h" +#include "llvm/CodeGen/CodeGenPrepare.h" #include "llvm/CodeGen/DwarfEHPrepare.h" #include "llvm/CodeGen/ExpandLargeDivRem.h" #include "llvm/CodeGen/ExpandLargeFpConvert.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 82ce040c6496..a8965335b2ea 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -229,6 +229,7 @@ CGSCC_PASS_WITH_PARAMS( FUNCTION_ANALYSIS("aa", AAManager()) FUNCTION_ANALYSIS("access-info", LoopAccessAnalysis()) FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis()) +FUNCTION_ANALYSIS("bb-sections-profile-reader", BasicBlockSectionsProfileReaderAnalysis(TM)) FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis()) FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis()) FUNCTION_ANALYSIS("cycles", CycleAnalysis()) @@ -290,6 +291,7 @@ FUNCTION_PASS("break-crit-edges", BreakCriticalEdgesPass()) FUNCTION_PASS("callbrprepare", CallBrPreparePass()) FUNCTION_PASS("callsite-splitting", CallSiteSplittingPass()) FUNCTION_PASS("chr", ControlHeightReductionPass()) +FUNCTION_PASS("codegenprepare", CodeGenPreparePass(TM)) FUNCTION_PASS("consthoist", ConstantHoistingPass()) FUNCTION_PASS("constraint-elimination", ConstraintEliminationPass()) FUNCTION_PASS("coro-elide", CoroElidePass()) diff --git a/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll b/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll index 92f29dac13fa..10b594978bee 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare < %s -S | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' < %s -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll index f4e9551259e4..4d085869de24 100644 --- a/llvm/test/CodeGen/AArch64/and-sink.ll +++ b/llvm/test/CodeGen/AArch64/and-sink.ll @@ -1,6 +1,6 @@ ; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs < %s | FileCheck %s -; RUN: opt -S -codegenprepare -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s -; RUN: opt -S -codegenprepare -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s +; RUN: opt -S -passes='require,function(codegenprepare)' -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s @A = dso_local global i32 zeroinitializer @B = dso_local global i32 zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll index 6041904dc0f3..2b42a3f29a72 100644 --- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll +++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s ; RUN: llc < %s -mtriple=arm64-eabi | FileCheck --check-prefix=LLC %s %struct.X = type { i8, i8, [2 x i8] } diff --git a/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll b/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll index 23cbad0d15b4..78a0bb4982b6 100644 --- a/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll +++ b/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll @@ -1,6 +1,6 @@ -; RUN: opt -codegenprepare < %s -mtriple=aarch64-apple-ios -S | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS -; RUN: opt -codegenprepare < %s -mtriple=aarch64-apple-ios -S -stress-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS -; RUN: opt -codegenprepare < %s -mtriple=aarch64-apple-ios -S -disable-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=DISABLE +; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-apple-ios -S | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS +; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-apple-ios -S -stress-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS +; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-apple-ios -S -disable-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=DISABLE ; CodeGenPrepare should move the zext into the block with the load ; so that SelectionDAG can select it with the load. diff --git a/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll b/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll index 964c669439d4..c33159163901 100644 --- a/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll +++ b/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -codegenprepare -mtriple=arm64_32-apple-ios %s -S -o - | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=arm64_32-apple-ios %s -S -o - | FileCheck %s define void @test_simple_sink(ptr %base, i64 %offset) { ; CHECK-LABEL: define void @test_simple_sink( diff --git a/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll b/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll index 98b820709e82..dfcedc8dd984 100644 --- a/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll +++ b/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll @@ -1,5 +1,5 @@ ; Checks that case when GEP is bound to trivial PHI node is correctly handled. -; RUN: opt %s -mtriple=aarch64-linux-gnu -codegenprepare -S -o - | FileCheck %s +; RUN: opt %s -mtriple=aarch64-linux-gnu -passes='require,function(codegenprepare)' -S -o - | FileCheck %s ; CHECK: define void @crash(ptr %s, i32 %n) { ; CHECK-NEXT: entry: diff --git a/llvm/test/CodeGen/AArch64/convertphitype.ll b/llvm/test/CodeGen/AArch64/convertphitype.ll index a5fc46d2abca..b723b470266a 100644 --- a/llvm/test/CodeGen/AArch64/convertphitype.ll +++ b/llvm/test/CodeGen/AArch64/convertphitype.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -cgp-optimize-phi-types %s -S | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -cgp-optimize-phi-types %s -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll b/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll index e6ab52dc9e61..a45f3733dbda 100644 --- a/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll +++ b/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -mtriple=aarch64 -codegenprepare -S < %s | FileCheck %s +; RUN: opt -mtriple=aarch64 -passes='require,function(codegenprepare)' -S < %s | FileCheck %s ; This test intends to check vector promotion for scalable vector. Current target lowering ; rejects scalable vector before reaching getConstantVector() in CodeGenPrepare. This test diff --git a/llvm/test/CodeGen/AArch64/sve-vscale.ll b/llvm/test/CodeGen/AArch64/sve-vscale.ll index fa48808ff7f8..7214c98cc318 100644 --- a/llvm/test/CodeGen/AArch64/sve-vscale.ll +++ b/llvm/test/CodeGen/AArch64/sve-vscale.ll @@ -1,5 +1,5 @@ ; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=0 < %s | FileCheck %s -; RUN: opt -mtriple=aarch64 -codegenprepare -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s +; RUN: opt -mtriple=aarch64 -passes='require,function(codegenprepare)' -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s ; ; RDVL diff --git a/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll b/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll index c80aa82ef9a8..bf3082d99c66 100644 --- a/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll +++ b/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt -codegenprepare -S -o - %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -o - %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll index 3005b17e0524..88a8e7cc4220 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck --check-prefixes=OPT,OPT-GFX7 %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga < %s | FileCheck --check-prefixes=OPT,OPT-GFX8 %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck --check-prefixes=OPT,OPT-GFX9 %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx1030 < %s | FileCheck --check-prefixes=OPT,OPT-GFX10 %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck --check-prefixes=OPT,OPT-GFX7 %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tonga < %s | FileCheck --check-prefixes=OPT,OPT-GFX8 %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck --check-prefixes=OPT,OPT-GFX9 %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=gfx1030 < %s | FileCheck --check-prefixes=OPT,OPT-GFX10 %s ; RUN: llc -march=amdgcn -mcpu=bonaire < %s | FileCheck --check-prefix=GFX7 %s ; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=GFX8 %s diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll index aee6f0e82d25..1588dde19cfb 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=OPT %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=OPT %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GCN %s ; Make sure we match the addressing mode offset of csub intrinsics across blocks. diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll index 494b4b5c48ba..ac50fb86c96f 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=OPT %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=OPT %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=GCN %s ; Make sure we match the addressing mode offset of globla.atomic.fadd intrinsics across blocks. diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll index 0e61547c27b4..65a604c98c4b 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll @@ -1,7 +1,7 @@ -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI -check-prefix=OPT-SICIVI %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI -check-prefix=OPT-SICIVI %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI -check-prefix=OPT-SICIVI %s -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-GFX9 %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI -check-prefix=OPT-SICIVI %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI -check-prefix=OPT-SICIVI %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI -check-prefix=OPT-SICIVI %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-GFX9 %s ; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SICIVI %s ; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=SICIVI %s ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s diff --git a/llvm/test/CodeGen/ARM/vector-promotion.ll b/llvm/test/CodeGen/ARM/vector-promotion.ll index 3e314306ff08..f4a2a4a4521e 100644 --- a/llvm/test/CodeGen/ARM/vector-promotion.ll +++ b/llvm/test/CodeGen/ARM/vector-promotion.ll @@ -1,5 +1,5 @@ -; RUN: opt -codegenprepare -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-NORMAL %s -; RUN: opt -codegenprepare -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S -stress-cgp-store-extract | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-STRESS %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-NORMAL %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S -stress-cgp-store-extract | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-STRESS %s ; RUN: llc -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon | FileCheck --check-prefix=ASM %s ; IR-BOTH-LABEL: @simpleOneInstructionPromotion diff --git a/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll b/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll index b02bdc3b5724..2a8d05c4f26e 100644 --- a/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll +++ b/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s ; REQUIRES: aarch64-registered-target ; Check that we don't give up if unable to sink the first argument. diff --git a/llvm/test/CodeGen/Generic/addr-use-count.ll b/llvm/test/CodeGen/Generic/addr-use-count.ll index 00943b5a58e2..5c4c0c618794 100644 --- a/llvm/test/CodeGen/Generic/addr-use-count.ll +++ b/llvm/test/CodeGen/Generic/addr-use-count.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s ; REQUIRES: aarch64-registered-target ; Test that `%addr` is sunk, after we've increased limit on the number of the memory uses to scan. diff --git a/llvm/test/CodeGen/X86/callbr-codegenprepare.ll b/llvm/test/CodeGen/X86/callbr-codegenprepare.ll index 854cc4bf7a9c..d2380a730b5b 100644 --- a/llvm/test/CodeGen/X86/callbr-codegenprepare.ll +++ b/llvm/test/CodeGen/X86/callbr-codegenprepare.ll @@ -1,4 +1,4 @@ -;; RUN: opt -S -codegenprepare < %s | FileCheck %s +;; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s ;; Ensure that codegenprepare (via InstSimplify) doesn't eliminate the ;; phi here (which would cause a module verification error). diff --git a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll index 6e95c91e7398..c611e89f2786 100644 --- a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll +++ b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare %s -o - | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' %s -o - | FileCheck %s ; This file tests the different cases what are involved when codegen prepare ; tries to get sign/zero extension out of the way of addressing mode. ; This tests require an actual target as addressing mode decisions depends diff --git a/llvm/test/CodeGen/X86/codegen-prepare-extload.ll b/llvm/test/CodeGen/X86/codegen-prepare-extload.ll index ff0ac5bfd6a7..ce87985fb3a0 100644 --- a/llvm/test/CodeGen/X86/codegen-prepare-extload.ll +++ b/llvm/test/CodeGen/X86/codegen-prepare-extload.ll @@ -1,9 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s ; RUN: llc < %s -mtriple=x86_64-win64 | FileCheck %s -; RUN: opt -codegenprepare < %s -mtriple=x86_64-apple-macosx -S | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS -; RUN: opt -codegenprepare < %s -mtriple=x86_64-apple-macosx -S -stress-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS -; RUN: opt -codegenprepare < %s -mtriple=x86_64-apple-macosx -S -disable-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=DISABLE +; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=x86_64-apple-macosx -S | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS +; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=x86_64-apple-macosx -S -stress-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS +; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=x86_64-apple-macosx -S -disable-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=DISABLE ; rdar://7304838 ; CodeGenPrepare should move the zext into the block with the load diff --git a/llvm/test/CodeGen/X86/convertphitype.ll b/llvm/test/CodeGen/X86/convertphitype.ll index df01612252bf..6c77236fc698 100644 --- a/llvm/test/CodeGen/X86/convertphitype.ll +++ b/llvm/test/CodeGen/X86/convertphitype.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -cgp-optimize-phi-types=true %s -S | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -cgp-optimize-phi-types=true %s -S | FileCheck %s target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" target triple = "i386-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll b/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll index 6b953e300425..ac23de49d3bd 100644 --- a/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll +++ b/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S -codegenprepare %s -o - | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' %s -o - | FileCheck %s target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/X86/pr58538.ll b/llvm/test/CodeGen/X86/pr58538.ll index 6c4103718950..7bae2594fc02 100644 --- a/llvm/test/CodeGen/X86/pr58538.ll +++ b/llvm/test/CodeGen/X86/pr58538.ll @@ -1,5 +1,5 @@ -; RUN: opt -codegenprepare -mtriple=x86_64 %s -S -o - | FileCheck %s -; RUN: opt -codegenprepare -mtriple=i386 %s -S -o - | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64 %s -S -o - | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=i386 %s -S -o - | FileCheck %s define i32 @f(i32 %0) { ; CHECK-LABEL: @f diff --git a/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll b/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll index 48440558283d..75bbae1050d6 100644 --- a/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll +++ b/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s -; RUN: opt -S -codegenprepare %s -mtriple=x86_64-apple-darwin -o - | FileCheck %s --check-prefix OPT +; RUN: opt -S -passes='require,function(codegenprepare)' %s -mtriple=x86_64-apple-darwin -o - | FileCheck %s --check-prefix OPT ; Teach CGP to dup returns to enable tail call optimization. ; rdar://9147433 diff --git a/llvm/test/CodeGen/X86/tailcall-extract.ll b/llvm/test/CodeGen/X86/tailcall-extract.ll index c3597a8e5b99..7a6c75c44ca7 100644 --- a/llvm/test/CodeGen/X86/tailcall-extract.ll +++ b/llvm/test/CodeGen/X86/tailcall-extract.ll @@ -1,5 +1,5 @@ ; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s -; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s --check-prefix OPT +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s --check-prefix OPT ; The exit block containing extractvalue can be duplicated into the BB diff --git a/llvm/test/DebugInfo/ARM/salvage-debug-info.ll b/llvm/test/DebugInfo/ARM/salvage-debug-info.ll index 3717abada42e..1564a80ded0e 100644 --- a/llvm/test/DebugInfo/ARM/salvage-debug-info.ll +++ b/llvm/test/DebugInfo/ARM/salvage-debug-info.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S %s -o - | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S %s -o - | FileCheck %s ; typedef struct info { ; unsigned long long size; ; } info_t; diff --git a/llvm/test/DebugInfo/X86/zextload.ll b/llvm/test/DebugInfo/X86/zextload.ll index 888e230c258d..05d92b09c20c 100644 --- a/llvm/test/DebugInfo/X86/zextload.ll +++ b/llvm/test/DebugInfo/X86/zextload.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s ; ; This test case was generated from the following source code: ; diff --git a/llvm/test/Other/codegenprepare-and-debug.ll b/llvm/test/Other/codegenprepare-and-debug.ll index 9023e8f21997..95f48c630efb 100644 --- a/llvm/test/Other/codegenprepare-and-debug.ll +++ b/llvm/test/Other/codegenprepare-and-debug.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s ; RUN: opt -strip-debug -codegenprepare -S < %s | FileCheck %s ; REQUIRES: x86-registered-target diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll index 91194864c013..25d4492f4c16 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s @_MergedGlobals = external dso_local global <{ i32, i32 }>, align 4 diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll index adb13f1a4c9f..de7eeada6024 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -mtriple=aarch64-linux %s | FileCheck -enable-var-scope %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck -enable-var-scope %s ; Test for CodeGenPrepare::optimizeLoadExt(): simple case: two loads ; feeding a phi that zext's each loaded value. diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll index 0114d7f9f409..469d818af28f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll index e4c5b4ceee54..6444f6adcdcc 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll index 4caf6d0dc893..f72679f55e11 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S < %s | FileCheck %s -; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG ; Subset of tests from llvm/tests/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll ; to test shouldFormOverflowOp on SPARC, where it is not profitable to create @@ -167,5 +167,5 @@ define i1 @usubo_ult_i64_math_overflow_used(i64 %x, i64 %y, ptr %p) { ret i1 %ov } -; Check that every instruction inserted by -codegenprepare has a debug location. +; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll index 73322836d1b8..f170c8ff18c1 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S --codegenprepare < %s | FileCheck %s +; RUN: opt -S --passes='require,function(codegenprepare)' < %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll index 6a8b5733889e..fa53d536fa38 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -mtriple=arm64-apple-ios7.0 %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=arm64-apple-ios7.0 %s | FileCheck %s %foo = type { i8 } diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll index 60b1e81e3dcd..8999e8d901ad 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S %s | FileCheck %s target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" target triple = "arm64-apple-ios" diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll index acd40d744f71..20a5a9ededac 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn--amdhsa < %s | FileCheck %s define amdgpu_kernel void @test_sink_as999_small_max_mubuf_offset(ptr addrspace(999) %out, ptr addrspace(999) %in) { ; CHECK-LABEL: @test_sink_as999_small_max_mubuf_offset( diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll index 63098ab098a2..c64d5a357f3a 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown < %s | FileCheck -check-prefix=ASC -check-prefix=COMMON %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown < %s | FileCheck -check-prefix=ASC -check-prefix=COMMON %s ; COMMON-LABEL: @test_sink_ptrtoint_asc( ; ASC: addrspacecast diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll index 7e6020629df8..77e79a902f3b 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S -codegenprepare -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn--amdhsa < %s | FileCheck %s define i64 @no_sink_local_to_flat(i1 %pred, ptr addrspace(3) %ptr) { ; CHECK-LABEL: define i64 @no_sink_local_to_flat( diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll index 996cab1d1d2c..ff5cef7e781f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv8.1m.main-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll index 52c06fd52b7b..f84491933b25 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S %s -o - | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S %s -o - | FileCheck %s target triple = "thumbv7-apple-ios7.0.0" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll index ae76dbda4aa1..9bec9b53ed0c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -mtriple=arm7-unknown-unknown -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=arm7-unknown-unknown -S < %s | FileCheck %s declare void @llvm.memcpy.p0.p0.i32(ptr, ptr, i32, i1) nounwind declare void @llvm.memmove.p0.p0.i32(ptr, ptr, i32, i1) nounwind diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll index 3fbc21331410..2f32ef94f82c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv8m.main-arm-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll index 838486aa2486..49cffe286a54 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -mtriple=thumbv7m -disable-complex-addr-modes=false -addr-sink-new-select=true -addr-sink-new-phis=true < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=thumbv7m -disable-complex-addr-modes=false -addr-sink-new-select=true -addr-sink-new-phis=true < %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll index cd2087d94149..69c8b92a3558 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv6m-arm-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll index 76b119fe36aa..3f113e6ea163 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s target triple = "armv8m.main-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll index 94adf1970938..b8a45e847afc 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll index c571da4411e7..d46aaf51a59c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll index 21e47c614ad0..9ec533c2e653 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll index 424f7c3b0271..463fa0d487a7 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll index 17b0dbf81ac2..af9b5ccd7287 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll index 374f30dba508..7ee7f0550318 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll index 79aebb9c247a..65177d5ae3d7 100644 --- a/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll +++ b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare -mtriple=powerpc64-unknown-linux-gnu -data-layout="E-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=BE %s -; RUN: opt -S -codegenprepare -mtriple=powerpc64le-unknown-linux-gnu -data-layout="e-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=LE %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=powerpc64-unknown-linux-gnu -data-layout="E-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=BE %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=powerpc64le-unknown-linux-gnu -data-layout="e-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=LE %s define void @split_store_align1(float %x, ptr %p) { ; BE-LABEL: @split_store_align1( diff --git a/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll b/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll index 130401d78171..863b0b4ad26f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll +++ b/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare -mtriple=riscv32 %s \ +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv32 %s \ ; RUN: | FileCheck --check-prefixes=CHECK,NOZBS %s -; RUN: opt -S -codegenprepare -mtriple=riscv32 -mattr=+zbs %s \ +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv32 -mattr=+zbs %s \ ; RUN: | FileCheck --check-prefixes=CHECK,ZBS %s -; RUN: opt -S -codegenprepare -mtriple=riscv64 %s \ +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv64 %s \ ; RUN: | FileCheck --check-prefixes=CHECK,NOZBS %s -; RUN: opt -S -codegenprepare -mtriple=riscv64 -mattr=zbs %s \ +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv64 -mattr=zbs %s \ ; RUN: | FileCheck --check-prefixes=CHECK,ZBS %s @A = global i32 zeroinitializer diff --git a/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll b/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll index c70112e91ebd..00ad32e96748 100644 --- a/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll +++ b/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target triple = "riscv64-unknown-unknown" diff --git a/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll index 7525ae14fa35..ec60238cbf92 100644 --- a/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S < %s | FileCheck %s -; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG ; Subset of tests from llvm/tests/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll ; to test shouldFormOverflowOp on SPARC, where it is not profitable to create @@ -119,5 +119,5 @@ define i1 @usubo_ult_i64_math_overflow_used(i64 %x, i64 %y, ptr %p) { ret i1 %ov } -; Check that every instruction inserted by -codegenprepare has a debug location. +; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll b/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll index 614e80e3e89c..cb617671827e 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll @@ -1,5 +1,5 @@ -; RUN: opt -codegenprepare -S < %s | FileCheck %s -; RUN: opt -codegenprepare -S < %s --try-experimental-debuginfo-iterators | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S < %s --try-experimental-debuginfo-iterators | FileCheck %s ; The following target lines are needed for the test to exercise what it should. ; Without these lines, CodeGenPrepare does not try to sink the bitcasts. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll index 9eede8cf361b..bcb7edd6e91b 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll index 7433ff74bab5..9ce830c39477 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll b/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll index fbf294128163..c1b919d31e07 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll index 5f368faf46ee..3a3a5327da8d 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll @@ -1,10 +1,10 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s --check-prefix=SLOW -; RUN: opt -S -codegenprepare -mattr=+bmi < %s | FileCheck %s --check-prefix=FAST_TZ -; RUN: opt -S -codegenprepare -mattr=+lzcnt < %s | FileCheck %s --check-prefix=FAST_LZ +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s --check-prefix=SLOW +; RUN: opt -S -passes='require,function(codegenprepare)' -mattr=+bmi < %s | FileCheck %s --check-prefix=FAST_TZ +; RUN: opt -S -passes='require,function(codegenprepare)' -mattr=+lzcnt < %s | FileCheck %s --check-prefix=FAST_LZ -; RUN: opt -S -debugify -codegenprepare < %s | FileCheck %s --check-prefix=DEBUGINFO -; RUN: opt -S -debugify -codegenprepare --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=DEBUGINFO +; RUN: opt -S -enable-debugify -passes='require,function(codegenprepare)' < %s | FileCheck %s --check-prefix=DEBUGINFO +; RUN: opt -S -enable-debugify -passes='require,function(codegenprepare)' --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=DEBUGINFO target triple = "x86_64-unknown-unknown" target datalayout = "e-n32:64" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll b/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll index abdf54c6e542..e1d99fd932fb 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s define i32 @test1(ptr %d) nounwind { ; CHECK-LABEL: @test1( diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll index a0814e0a5f20..5349afc18d84 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -disable-cgp-branch-opts -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -disable-cgp-branch-opts -S < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll b/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll index c37227f5fa82..e9ecfd1615d4 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll index 41b1ac2c05fc..e62ba5d5a7f5 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s -; RUN: opt -S -codegenprepare -cgpp-huge-func=0 < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -cgpp-huge-func=0 < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll index 2cf98491acb9..7899477afdb2 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll index d2eae6954c5d..a2ea3f7e36a0 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s @exit_addr = constant ptr blockaddress(@gep_unmerging, %exit) @op1_addr = constant ptr blockaddress(@gep_unmerging, %op1) diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll b/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll index 3c81a4beb834..a0bac01d1165 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s @tmp = global i8 0 diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll b/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll index ea07a5fe9bc5..dbd5e87f2c28 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s ; REQUIRES: x86-registered-target target triple = "x86_64-pc-linux" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll b/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll index 2f42ad889b42..9d53855ada79 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll @@ -1,5 +1,5 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s -; RUN: opt -S -codegenprepare -addr-sink-using-gep=false < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -addr-sink-using-gep=false < %s | FileCheck %s ; This target data layout is modified to have a non-integral addrspace(1), ; in order to verify that codegenprepare does not try to introduce illegal diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll index be651d7eb004..aaf3df093468 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll index a324f6f44e5c..653f34635648 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S < %s | FileCheck %s -; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "x86_64-apple-darwin10.0.0" @@ -636,6 +636,6 @@ exit: ret void } -; Check that every instruction inserted by -codegenprepare has a debug location. +; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll index 3ef27c7c950f..51fba2229f3c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-windows-msvc" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll index eec9475a1c48..e9d0806235c9 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll index d75e5632ebb2..b6296871f575 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S -codegenprepare -mtriple=x86_64-unknown-unknown < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=x86_64-unknown-unknown < %s | FileCheck %s ; Make sure the nneg flag is dropped when lshr and zext are interchanged. define i8 @get(ptr %box, i32 %in) { diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll b/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll index 0366b7d7e6d2..eff88bba3773 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s declare void @llvm.assume(i1 noundef) nounwind willreturn diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll b/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll index 1d5e6ea0978a..6b7a122b3e26 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=x86_64-linux < %s | FileCheck %s ; ; Ensure that blocks that only contain @llvm.assume are removed completely ; during CodeGenPrepare. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll index a0f34a882d30..08dd77e9e4c3 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S < %s | FileCheck %s -; RUN: opt -debugify -codegenprepare -S < %s | FileCheck %s -check-prefix=DEBUG -; RUN: opt -debugify -codegenprepare -S < %s --try-experimental-debuginfo-iterators | FileCheck %s -check-prefix=DEBUG +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s | FileCheck %s -check-prefix=DEBUG +; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s --try-experimental-debuginfo-iterators | FileCheck %s -check-prefix=DEBUG target triple = "x86_64-unknown-unknown" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll index 45ddbe76c8f1..08e822f7e211 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll @@ -1,5 +1,5 @@ -; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-YES -; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=false -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NO +; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-YES +; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=false -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NO target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll index d5e69b9d802e..7660ee47fdbd 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll index 336421e4c500..076915028aef 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-select=true %s | FileCheck %s --check-prefix=CHECK +; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-select=true %s | FileCheck %s --check-prefix=CHECK target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll index 611ef908d706..6a6b029b67b5 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK +; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll index 97b11a2e1f1c..f75af606eff0 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll index f2e82212d0fa..a760f56b151f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -codegenprepare < %s | FileCheck %s -check-prefix=CHECK -check-prefix=GEP +; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s -check-prefix=CHECK -check-prefix=GEP target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll b/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll index c5d18ff50309..7f7f80910b48 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s ; Test that an invalid CFG is not created by splitIndirectCriticalEdges ; transformation when the 'target' block is a loop to itself. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll index 3bced480f31a..0335da94ea50 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -mtriple=x86_64-unknown-unknown -force-split-store -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-unknown-unknown -force-split-store -S < %s | FileCheck %s target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" target triple = "i686-w64-windows-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll b/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll index a8a6f7baf9b4..babaa08a959b 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll index 9dc88a100daa..dd47d5eb6cc4 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s ; The ret instruction can be duplicated into BB case2 even though there is an ; intermediate BB exit1 and call to llvm.assume. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll index 557974fcfe54..db7d960899ca 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll @@ -1,10 +1,10 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG define <4 x i32> @vector_variable_shift_right_v4i32(<4 x i1> %cond, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { ; AVX1-LABEL: @vector_variable_shift_right_v4i32( @@ -409,5 +409,5 @@ exit: declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) #1 -; Check that every instruction inserted by -codegenprepare has a debug location. +; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll index 482e822ea3d8..e0f04f77efa0 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll @@ -1,10 +1,10 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG define <4 x i32> @vector_variable_shift_right_v4i32(<4 x i1> %cond, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { ; AVX1-LABEL: @vector_variable_shift_right_v4i32( @@ -409,5 +409,5 @@ exit: declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) #1 -; Check that every instruction inserted by -codegenprepare has a debug location. +; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll b/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll index b26876e0e1e2..12230ec689cf 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s ; Check the idiomatic guard pattern to ensure it's lowered correctly. define void @test_guard(i1 %cond_0) { diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll index 72d1672eb4f7..ce1b6bd5ae63 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 -; RUN: opt -S -codegenprepare -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP -; RUN: opt -S -codegenprepare -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 -; RUN: opt -S -codegenprepare -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-darwin10.9.0" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll index c14918a6956f..9e82844dfc2f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -codegenprepare -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 -; RUN: opt -S -codegenprepare -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP -; RUN: opt -S -codegenprepare -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 -; RUN: opt -S -codegenprepare -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 +; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-darwin10.9.0" diff --git a/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll b/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll index 637040a0d56d..9550e748da6d 100644 --- a/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll +++ b/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; Eliminate the dead allocation instruction ; REQUIRES: arm-registered-target -; RUN: opt -codegenprepare < %s -S | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' < %s -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "armv7--linux-gnueabihf" diff --git a/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll b/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll index 608ad4c0a32f..d25b9c91aff6 100644 --- a/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll +++ b/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll @@ -1,5 +1,5 @@ ; REQUIRES: aarch64-registered-target -; RUN: opt -codegenprepare < %s -mtriple=aarch64-none-linux-gnu -S | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-none-linux-gnu -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll index 1a113ff16188..6e5ef1aa2539 100644 --- a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll +++ b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll @@ -1,5 +1,5 @@ ; REQUIRES: x86-registered-target -; RUN: opt -passes=hotcoldsplit -hotcoldsplit-threshold=0 < %s | opt -codegenprepare -S | FileCheck %s +; RUN: opt -passes=hotcoldsplit -hotcoldsplit-threshold=0 < %s | opt -passes='require,function(codegenprepare)' -S | FileCheck %s ; Test to ensure that split cold function gets 0 entry count profile ; metadata when compiling with pgo. diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll index ff8a804beeb5..a56efe8dd3f3 100644 --- a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll +++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll @@ -1,4 +1,4 @@ -; RUN: opt -codegenprepare -load-store-vectorizer %s -S -o - | FileCheck %s +; RUN: opt -passes='require,function(codegenprepare)' -passes=load-store-vectorizer %s -S -o - | FileCheck %s ; RUN: opt -passes=load-store-vectorizer %s -S -o - | FileCheck %s ; RUN: opt -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s diff --git a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll index a404220056c8..ef2ddbc33cee 100644 --- a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll +++ b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll @@ -1,7 +1,7 @@ ; REQUIRES: x86-registered-target -; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -codegenprepare | FileCheck %s -; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -codegenprepare -profile-unknown-in-special-section -partial-profile | FileCheck %s --check-prefix=UNKNOWN -; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -profile-sample-accurate -S | opt -S -codegenprepare | FileCheck %s --check-prefix=ACCURATE +; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -passes='require,function(codegenprepare)' | FileCheck %s +; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -passes='require,function(codegenprepare)' -profile-unknown-in-special-section -partial-profile | FileCheck %s --check-prefix=UNKNOWN +; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -profile-sample-accurate -S | opt -S -passes='require,function(codegenprepare)' | FileCheck %s --check-prefix=ACCURATE target triple = "x86_64-pc-linux-gnu" diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp index b6068513d230..c649e6ecddc0 100644 --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -426,7 +426,7 @@ int main(int argc, char **argv) { initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry); initializeSelectOptimizePass(Registry); initializeCallBrPreparePass(Registry); - initializeCodeGenPreparePass(Registry); + initializeCodeGenPrepareLegacyPassPass(Registry); initializeAtomicExpandPass(Registry); initializeWinEHPreparePass(Registry); initializeDwarfEHPrepareLegacyPassPass(Registry); -- GitLab From 59af659ee3c790d06cb5e2bf580e042547c24323 Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Fri, 5 Jan 2024 15:25:18 +0800 Subject: [PATCH 072/728] [X86][BF16] Try to use `f16` for lowering (#76901) This patch fixes BF16 32-bit ABI problem: https://godbolt.org/z/6dMnh8jGG --- llvm/lib/Target/X86/X86ISelLowering.cpp | 11 +- llvm/lib/Target/X86/X86ISelLowering.h | 10 - llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 37 +- llvm/test/CodeGen/X86/bfloat.ll | 1324 +++++++++---------- 4 files changed, 609 insertions(+), 773 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index a72b14c2e2aa..8c4f091c793d 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -7475,10 +7475,12 @@ static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) { static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) { MVT VT = Op.getSimpleValueType(); - MVT IVT = VT.changeVectorElementTypeToInteger(); + MVT IVT = + VT.changeVectorElementType(Subtarget.hasFP16() ? MVT::f16 : MVT::i16); SmallVector NewOps; for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) - NewOps.push_back(DAG.getBitcast(MVT::i16, Op.getOperand(I))); + NewOps.push_back(DAG.getBitcast(Subtarget.hasFP16() ? MVT::f16 : MVT::i16, + Op.getOperand(I))); SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps); return DAG.getBitcast(VT, Res); } @@ -21522,9 +21524,8 @@ SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op, RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::bf16); SDValue Res = - makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first; - return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, - DAG.getBitcast(MVT::i32, Res)); + makeLibCall(DAG, LC, MVT::f16, Op.getOperand(0), CallOptions, DL).first; + return DAG.getBitcast(MVT::i16, Res); } /// Depending on uarch and/or optimizing for size, we might prefer to use a diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 9bd1622cb0d3..32745400a38b 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1714,16 +1714,6 @@ namespace llvm { MachineBasicBlock *Entry, const SmallVectorImpl &Exits) const override; - bool splitValueIntoRegisterParts( - SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, - unsigned NumParts, MVT PartVT, std::optional CC) - const override; - - SDValue joinRegisterPartsIntoValue( - SelectionDAG & DAG, const SDLoc &DL, const SDValue *Parts, - unsigned NumParts, MVT PartVT, EVT ValueVT, - std::optional CC) const override; - bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override; bool mayBeEmittedAsTailCall(const CallInst *CI) const override; diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index b8b5421b9005..d75bd4171fde 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -127,6 +127,9 @@ MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, return getRegisterTypeForCallingConv(Context, CC, VT.changeVectorElementType(MVT::f16)); + if (VT == MVT::bf16) + return MVT::f16; + return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); } @@ -421,40 +424,6 @@ unsigned X86TargetLowering::getJumpTableEncoding() const { return TargetLowering::getJumpTableEncoding(); } -bool X86TargetLowering::splitValueIntoRegisterParts( - SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, - unsigned NumParts, MVT PartVT, std::optional CC) const { - bool IsABIRegCopy = CC.has_value(); - EVT ValueVT = Val.getValueType(); - if (IsABIRegCopy && ValueVT == MVT::bf16 && PartVT == MVT::f32) { - unsigned ValueBits = ValueVT.getSizeInBits(); - unsigned PartBits = PartVT.getSizeInBits(); - Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val); - Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val); - Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); - Parts[0] = Val; - return true; - } - return false; -} - -SDValue X86TargetLowering::joinRegisterPartsIntoValue( - SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, - MVT PartVT, EVT ValueVT, std::optional CC) const { - bool IsABIRegCopy = CC.has_value(); - if (IsABIRegCopy && ValueVT == MVT::bf16 && PartVT == MVT::f32) { - unsigned ValueBits = ValueVT.getSizeInBits(); - unsigned PartBits = PartVT.getSizeInBits(); - SDValue Val = Parts[0]; - - Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val); - Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val); - Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); - return Val; - } - return SDValue(); -} - bool X86TargetLowering::useSoftFloat() const { return Subtarget.useSoftFloat(); } diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll index 7ef362619d5f..b309f47e4b71 100644 --- a/llvm/test/CodeGen/X86/bfloat.ll +++ b/llvm/test/CodeGen/X86/bfloat.ll @@ -3,7 +3,7 @@ ; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,SSE2 ; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16,avx512vl | FileCheck %s --check-prefixes=CHECK,AVX,F16,BF16 ; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16,avx512fp16,avx512vl | FileCheck %s --check-prefixes=CHECK,AVX,F16,FP16 -; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avxneconvert,f16c | FileCheck %s --check-prefixes=CHECK,AVX,AVXNC +; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avxneconvert,f16c | FileCheck %s --check-prefixes=CHECK,AVX,BF16,AVXNC define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind { ; X86-LABEL: add: @@ -22,10 +22,7 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind { ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: movw %ax, (%esi) +; X86-NEXT: vmovsh %xmm0, (%esi) ; X86-NEXT: addl $8, %esp ; X86-NEXT: popl %esi ; X86-NEXT: retl @@ -42,27 +39,42 @@ define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind { ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movw %ax, (%rbx) ; SSE2-NEXT: popq %rbx ; SSE2-NEXT: retq ; -; AVX-LABEL: add: -; AVX: # %bb.0: -; AVX-NEXT: pushq %rbx -; AVX-NEXT: movq %rdx, %rbx -; AVX-NEXT: movzwl (%rsi), %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: movzwl (%rdi), %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm1 -; AVX-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVX-NEXT: callq __truncsfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: movw %ax, (%rbx) -; AVX-NEXT: popq %rbx -; AVX-NEXT: retq +; BF16-LABEL: add: +; BF16: # %bb.0: +; BF16-NEXT: pushq %rbx +; BF16-NEXT: movq %rdx, %rbx +; BF16-NEXT: movzwl (%rsi), %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: movzwl (%rdi), %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm1 +; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; BF16-NEXT: callq __truncsfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, (%rbx) +; BF16-NEXT: popq %rbx +; BF16-NEXT: retq +; +; FP16-LABEL: add: +; FP16: # %bb.0: +; FP16-NEXT: pushq %rbx +; FP16-NEXT: movq %rdx, %rbx +; FP16-NEXT: movzwl (%rsi), %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm0 +; FP16-NEXT: movzwl (%rdi), %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm1 +; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; FP16-NEXT: callq __truncsfbf2@PLT +; FP16-NEXT: vmovsh %xmm0, (%rbx) +; FP16-NEXT: popq %rbx +; FP16-NEXT: retq %a = load bfloat, ptr %pa %b = load bfloat, ptr %pb %add = fadd bfloat %a, %b @@ -89,8 +101,8 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind { ; SSE2-LABEL: add2: ; SSE2: # %bb.0: ; SSE2-NEXT: pushq %rax -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: movd %xmm1, %ecx +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm1, %ecx ; SSE2-NEXT: shll $16, %ecx ; SSE2-NEXT: movd %ecx, %xmm1 ; SSE2-NEXT: shll $16, %eax @@ -100,19 +112,33 @@ define bfloat @add2(bfloat %a, bfloat %b) nounwind { ; SSE2-NEXT: popq %rax ; SSE2-NEXT: retq ; -; AVX-LABEL: add2: -; AVX: # %bb.0: -; AVX-NEXT: pushq %rax -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: vmovd %xmm1, %ecx -; AVX-NEXT: shll $16, %ecx -; AVX-NEXT: vmovd %ecx, %xmm0 -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm1 -; AVX-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVX-NEXT: callq __truncsfbf2@PLT -; AVX-NEXT: popq %rax -; AVX-NEXT: retq +; BF16-LABEL: add2: +; BF16: # %bb.0: +; BF16-NEXT: pushq %rax +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: vpextrw $0, %xmm1, %ecx +; BF16-NEXT: shll $16, %ecx +; BF16-NEXT: vmovd %ecx, %xmm0 +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm1 +; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; BF16-NEXT: callq __truncsfbf2@PLT +; BF16-NEXT: popq %rax +; BF16-NEXT: retq +; +; FP16-LABEL: add2: +; FP16: # %bb.0: +; FP16-NEXT: pushq %rax +; FP16-NEXT: vmovw %xmm0, %eax +; FP16-NEXT: vmovw %xmm1, %ecx +; FP16-NEXT: shll $16, %ecx +; FP16-NEXT: vmovd %ecx, %xmm0 +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm1 +; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; FP16-NEXT: callq __truncsfbf2@PLT +; FP16-NEXT: popq %rax +; FP16-NEXT: retq %add = fadd bfloat %a, %b ret bfloat %add } @@ -123,22 +149,18 @@ define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind { ; X86-NEXT: pushl %ebx ; X86-NEXT: pushl %edi ; X86-NEXT: pushl %esi -; X86-NEXT: subl $32, %esp +; X86-NEXT: subl $16, %esp ; X86-NEXT: movl {{[0-9]+}}(%esp), %esi ; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; X86-NEXT: vmovsd %xmm0, (%esp) ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %edi +; X86-NEXT: vmovw %xmm0, %edi ; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; X86-NEXT: vmovsd %xmm0, (%esp) ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %eax +; X86-NEXT: vmovw %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: shll $16, %edi @@ -146,14 +168,12 @@ define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind { ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %eax +; X86-NEXT: vmovw %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 ; X86-NEXT: vmovsd %xmm0, (%esi) -; X86-NEXT: addl $32, %esp +; X86-NEXT: addl $16, %esp ; X86-NEXT: popl %esi ; X86-NEXT: popl %edi ; X86-NEXT: popl %ebx @@ -168,17 +188,17 @@ define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind { ; SSE2-NEXT: movq %rsi, %r14 ; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movd %ebp, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: cvtss2sd %xmm0, %xmm0 @@ -188,34 +208,63 @@ define void @add_double(ptr %pa, ptr %pb, ptr %pc) nounwind { ; SSE2-NEXT: popq %rbp ; SSE2-NEXT: retq ; -; AVX-LABEL: add_double: -; AVX: # %bb.0: -; AVX-NEXT: pushq %rbp -; AVX-NEXT: pushq %r14 -; AVX-NEXT: pushq %rbx -; AVX-NEXT: movq %rdx, %rbx -; AVX-NEXT: movq %rsi, %r14 -; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; AVX-NEXT: callq __truncdfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %ebp -; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; AVX-NEXT: callq __truncdfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: shll $16, %ebp -; AVX-NEXT: vmovd %ebp, %xmm1 -; AVX-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVX-NEXT: callq __truncsfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 -; AVX-NEXT: vmovsd %xmm0, (%rbx) -; AVX-NEXT: popq %rbx -; AVX-NEXT: popq %r14 -; AVX-NEXT: popq %rbp -; AVX-NEXT: retq +; BF16-LABEL: add_double: +; BF16: # %bb.0: +; BF16-NEXT: pushq %rbp +; BF16-NEXT: pushq %r14 +; BF16-NEXT: pushq %rbx +; BF16-NEXT: movq %rdx, %rbx +; BF16-NEXT: movq %rsi, %r14 +; BF16-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; BF16-NEXT: callq __truncdfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, %ebp +; BF16-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; BF16-NEXT: callq __truncdfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: shll $16, %ebp +; BF16-NEXT: vmovd %ebp, %xmm1 +; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; BF16-NEXT: callq __truncsfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 +; BF16-NEXT: vmovsd %xmm0, (%rbx) +; BF16-NEXT: popq %rbx +; BF16-NEXT: popq %r14 +; BF16-NEXT: popq %rbp +; BF16-NEXT: retq +; +; FP16-LABEL: add_double: +; FP16: # %bb.0: +; FP16-NEXT: pushq %rbp +; FP16-NEXT: pushq %r14 +; FP16-NEXT: pushq %rbx +; FP16-NEXT: movq %rdx, %rbx +; FP16-NEXT: movq %rsi, %r14 +; FP16-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovw %xmm0, %ebp +; FP16-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovw %xmm0, %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm0 +; FP16-NEXT: shll $16, %ebp +; FP16-NEXT: vmovd %ebp, %xmm1 +; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; FP16-NEXT: callq __truncsfbf2@PLT +; FP16-NEXT: vmovw %xmm0, %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm0 +; FP16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 +; FP16-NEXT: vmovsd %xmm0, (%rbx) +; FP16-NEXT: popq %rbx +; FP16-NEXT: popq %r14 +; FP16-NEXT: popq %rbp +; FP16-NEXT: retq %la = load double, ptr %pa %a = fptrunc double %la to bfloat %lb = load double, ptr %pb @@ -230,19 +279,15 @@ define double @add_double2(double %da, double %db) nounwind { ; X86-LABEL: add_double2: ; X86: # %bb.0: ; X86-NEXT: pushl %esi -; X86-NEXT: subl $40, %esp +; X86-NEXT: subl $24, %esp ; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; X86-NEXT: vmovsd %xmm0, (%esp) ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %esi +; X86-NEXT: vmovw %xmm0, %esi ; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; X86-NEXT: vmovsd %xmm0, (%esp) ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %eax +; X86-NEXT: vmovw %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: shll $16, %esi @@ -250,15 +295,13 @@ define double @add_double2(double %da, double %db) nounwind { ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %eax +; X86-NEXT: vmovw %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 ; X86-NEXT: vmovsd %xmm0, {{[0-9]+}}(%esp) ; X86-NEXT: fldl {{[0-9]+}}(%esp) -; X86-NEXT: addl $40, %esp +; X86-NEXT: addl $24, %esp ; X86-NEXT: popl %esi ; X86-NEXT: retl ; @@ -268,18 +311,18 @@ define double @add_double2(double %da, double %db) nounwind { ; SSE2-NEXT: subq $16, %rsp ; SSE2-NEXT: movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd %ebx, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: cvtss2sd %xmm0, %xmm0 @@ -287,30 +330,55 @@ define double @add_double2(double %da, double %db) nounwind { ; SSE2-NEXT: popq %rbx ; SSE2-NEXT: retq ; -; AVX-LABEL: add_double2: -; AVX: # %bb.0: -; AVX-NEXT: pushq %rbx -; AVX-NEXT: subq $16, %rsp -; AVX-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; AVX-NEXT: callq __truncdfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %ebx -; AVX-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload -; AVX-NEXT: # xmm0 = mem[0],zero -; AVX-NEXT: callq __truncdfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: shll $16, %ebx -; AVX-NEXT: vmovd %ebx, %xmm1 -; AVX-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVX-NEXT: callq __truncsfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 -; AVX-NEXT: addq $16, %rsp -; AVX-NEXT: popq %rbx -; AVX-NEXT: retq +; BF16-LABEL: add_double2: +; BF16: # %bb.0: +; BF16-NEXT: pushq %rbx +; BF16-NEXT: subq $16, %rsp +; BF16-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; BF16-NEXT: callq __truncdfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, %ebx +; BF16-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload +; BF16-NEXT: # xmm0 = mem[0],zero +; BF16-NEXT: callq __truncdfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: shll $16, %ebx +; BF16-NEXT: vmovd %ebx, %xmm1 +; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; BF16-NEXT: callq __truncsfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 +; BF16-NEXT: addq $16, %rsp +; BF16-NEXT: popq %rbx +; BF16-NEXT: retq +; +; FP16-LABEL: add_double2: +; FP16: # %bb.0: +; FP16-NEXT: pushq %rbx +; FP16-NEXT: subq $16, %rsp +; FP16-NEXT: vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovw %xmm0, %ebx +; FP16-NEXT: vmovsd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Reload +; FP16-NEXT: # xmm0 = mem[0],zero +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovw %xmm0, %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm0 +; FP16-NEXT: shll $16, %ebx +; FP16-NEXT: vmovd %ebx, %xmm1 +; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 +; FP16-NEXT: callq __truncsfbf2@PLT +; FP16-NEXT: vmovw %xmm0, %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm0 +; FP16-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 +; FP16-NEXT: addq $16, %rsp +; FP16-NEXT: popq %rbx +; FP16-NEXT: retq %a = fptrunc double %da to bfloat %b = fptrunc double %db to bfloat %add = fadd bfloat %a, %b @@ -331,10 +399,7 @@ define void @add_constant(ptr %pa, ptr %pc) nounwind { ; X86-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: movw %ax, (%esi) +; X86-NEXT: vmovsh %xmm0, (%esi) ; X86-NEXT: addl $8, %esp ; X86-NEXT: popl %esi ; X86-NEXT: retl @@ -348,24 +413,36 @@ define void @add_constant(ptr %pa, ptr %pc) nounwind { ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movw %ax, (%rbx) ; SSE2-NEXT: popq %rbx ; SSE2-NEXT: retq ; -; AVX-LABEL: add_constant: -; AVX: # %bb.0: -; AVX-NEXT: pushq %rbx -; AVX-NEXT: movq %rsi, %rbx -; AVX-NEXT: movzwl (%rdi), %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: callq __truncsfbf2@PLT -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: movw %ax, (%rbx) -; AVX-NEXT: popq %rbx -; AVX-NEXT: retq +; BF16-LABEL: add_constant: +; BF16: # %bb.0: +; BF16-NEXT: pushq %rbx +; BF16-NEXT: movq %rsi, %rbx +; BF16-NEXT: movzwl (%rdi), %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; BF16-NEXT: callq __truncsfbf2@PLT +; BF16-NEXT: vpextrw $0, %xmm0, (%rbx) +; BF16-NEXT: popq %rbx +; BF16-NEXT: retq +; +; FP16-LABEL: add_constant: +; FP16: # %bb.0: +; FP16-NEXT: pushq %rbx +; FP16-NEXT: movq %rsi, %rbx +; FP16-NEXT: movzwl (%rdi), %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm0 +; FP16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; FP16-NEXT: callq __truncsfbf2@PLT +; FP16-NEXT: vmovsh %xmm0, (%rbx) +; FP16-NEXT: popq %rbx +; FP16-NEXT: retq %a = load bfloat, ptr %pa %add = fadd bfloat %a, 1.0 store bfloat %add, ptr %pc @@ -388,7 +465,7 @@ define bfloat @add_constant2(bfloat %a) nounwind { ; SSE2-LABEL: add_constant2: ; SSE2: # %bb.0: ; SSE2-NEXT: pushq %rax -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 @@ -396,16 +473,27 @@ define bfloat @add_constant2(bfloat %a) nounwind { ; SSE2-NEXT: popq %rax ; SSE2-NEXT: retq ; -; AVX-LABEL: add_constant2: -; AVX: # %bb.0: -; AVX-NEXT: pushq %rax -; AVX-NEXT: vmovd %xmm0, %eax -; AVX-NEXT: shll $16, %eax -; AVX-NEXT: vmovd %eax, %xmm0 -; AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: callq __truncsfbf2@PLT -; AVX-NEXT: popq %rax -; AVX-NEXT: retq +; BF16-LABEL: add_constant2: +; BF16: # %bb.0: +; BF16-NEXT: pushq %rax +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: shll $16, %eax +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; BF16-NEXT: callq __truncsfbf2@PLT +; BF16-NEXT: popq %rax +; BF16-NEXT: retq +; +; FP16-LABEL: add_constant2: +; FP16: # %bb.0: +; FP16-NEXT: pushq %rax +; FP16-NEXT: vmovw %xmm0, %eax +; FP16-NEXT: shll $16, %eax +; FP16-NEXT: vmovd %eax, %xmm0 +; FP16-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; FP16-NEXT: callq __truncsfbf2@PLT +; FP16-NEXT: popq %rax +; FP16-NEXT: retq %add = fadd bfloat %a, 1.0 ret bfloat %add } @@ -449,7 +537,7 @@ define void @fold_ext_trunc(ptr %pa, ptr %pc) nounwind { define bfloat @fold_ext_trunc2(bfloat %a) nounwind { ; X86-LABEL: fold_ext_trunc2: ; X86: # %bb.0: -; X86-NEXT: flds {{[0-9]+}}(%esp) +; X86-NEXT: vmovsh {{[0-9]+}}(%esp), %xmm0 ; X86-NEXT: retl ; ; CHECK-LABEL: fold_ext_trunc2: @@ -464,159 +552,136 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; X86-LABEL: addv: ; X86: # %bb.0: ; X86-NEXT: subl $172, %esp -; X86-NEXT: vpextrw $1, %xmm1, %eax -; X86-NEXT: vmovdqa %xmm1, %xmm4 +; X86-NEXT: vmovw %xmm1, %eax +; X86-NEXT: vmovdqa %xmm1, %xmm3 ; X86-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm2 -; X86-NEXT: vpextrw $1, %xmm0, %eax -; X86-NEXT: vmovdqa %xmm0, %xmm3 +; X86-NEXT: vmovw %xmm0, %eax +; X86-NEXT: vmovdqa %xmm0, %xmm4 ; X86-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm2, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) -; X86-NEXT: vpextrw $2, %xmm4, %eax +; X86-NEXT: vpextrw $1, %xmm3, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 -; X86-NEXT: vpextrw $2, %xmm3, %eax +; X86-NEXT: vpextrw $1, %xmm4, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload ; X86-NEXT: # xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload -; X86-NEXT: vpextrw $3, %xmm0, %eax +; X86-NEXT: vpextrw $2, %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload -; X86-NEXT: vpextrw $3, %xmm1, %eax +; X86-NEXT: vpextrw $2, %xmm1, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload ; X86-NEXT: # xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload -; X86-NEXT: vpextrw $4, %xmm0, %eax +; X86-NEXT: vpextrw $3, %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload -; X86-NEXT: vpextrw $4, %xmm1, %eax +; X86-NEXT: vpextrw $3, %xmm1, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload ; X86-NEXT: # xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload -; X86-NEXT: vpextrw $5, %xmm0, %eax +; X86-NEXT: vpextrw $4, %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload -; X86-NEXT: vpextrw $5, %xmm1, %eax +; X86-NEXT: vpextrw $4, %xmm1, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload ; X86-NEXT: # xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload -; X86-NEXT: vpextrw $6, %xmm0, %eax +; X86-NEXT: vpextrw $5, %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload -; X86-NEXT: vpextrw $6, %xmm1, %eax +; X86-NEXT: vpextrw $5, %xmm1, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload ; X86-NEXT: # xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload -; X86-NEXT: vpextrw $7, %xmm0, %eax +; X86-NEXT: vpextrw $6, %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 ; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload -; X86-NEXT: vpextrw $7, %xmm1, %eax +; X86-NEXT: vpextrw $6, %xmm1, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload ; X86-NEXT: # xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss %xmm0, (%esp) -; X86-NEXT: vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload -; X86-NEXT: vmovw %xmm0, %eax +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload +; X86-NEXT: vpextrw $7, %xmm0, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm0 -; X86-NEXT: vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload -; X86-NEXT: vmovw %xmm1, %eax +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload +; X86-NEXT: vpextrw $7, %xmm1, %eax ; X86-NEXT: shll $16, %eax ; X86-NEXT: vmovd %eax, %xmm1 ; X86-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: vmovss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill +; X86-NEXT: vmovd {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; X86-NEXT: # xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovss %xmm0, (%esp) -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) +; X86-NEXT: vmovd %xmm0, (%esp) ; X86-NEXT: calll __truncsfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm1, %eax -; X86-NEXT: vmovd %eax, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $1, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $4, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vpinsrw $7, %eax, %xmm1, %xmm0 +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; X86-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload +; X86-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3] +; X86-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; X86-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] ; X86-NEXT: addl $172, %esp ; X86-NEXT: retl ; @@ -632,93 +697,92 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; SSE2-NEXT: movq %xmm0, %rcx ; SSE2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; SSE2-NEXT: movq %rcx, %rax -; SSE2-NEXT: shrq $32, %rax +; SSE2-NEXT: shrq $48, %rax ; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; SSE2-NEXT: movq %xmm1, %rdx ; SSE2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; SSE2-NEXT: movq %rdx, %rax -; SSE2-NEXT: shrq $32, %rax +; SSE2-NEXT: shrq $48, %rax ; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; SSE2-NEXT: movq %rcx, %rax -; SSE2-NEXT: shrq $48, %rax +; SSE2-NEXT: shrq $32, %rax ; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; SSE2-NEXT: movq %rdx, %rax -; SSE2-NEXT: shrq $48, %rax +; SSE2-NEXT: shrq $32, %rax ; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] -; SSE2-NEXT: movq %xmm0, %r12 -; SSE2-NEXT: movq %r12, %rax -; SSE2-NEXT: shrq $32, %rax -; SSE2-NEXT: movq %rax, (%rsp) # 8-byte Spill +; SSE2-NEXT: movq %xmm0, %r15 +; SSE2-NEXT: movq %r15, %rbx +; SSE2-NEXT: shrq $48, %rbx ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1] ; SSE2-NEXT: movq %xmm1, %r14 ; SSE2-NEXT: movq %r14, %rbp -; SSE2-NEXT: shrq $32, %rbp -; SSE2-NEXT: movq %r12, %r15 -; SSE2-NEXT: shrq $48, %r15 +; SSE2-NEXT: shrq $48, %rbp +; SSE2-NEXT: movq %r15, %r12 +; SSE2-NEXT: shrq $32, %r12 ; SSE2-NEXT: movq %r14, %r13 -; SSE2-NEXT: shrq $48, %r13 +; SSE2-NEXT: shrq $32, %r13 ; SSE2-NEXT: movl %r14d, %eax -; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000 +; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: movl %r12d, %eax -; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000 +; SSE2-NEXT: movl %r15d, %eax +; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx -; SSE2-NEXT: shll $16, %ebx -; SSE2-NEXT: shll $16, %r14d +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: movzwl %ax, %eax +; SSE2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; SSE2-NEXT: andl $-65536, %r14d # imm = 0xFFFF0000 ; SSE2-NEXT: movd %r14d, %xmm1 -; SSE2-NEXT: shll $16, %r12d -; SSE2-NEXT: movd %r12d, %xmm0 +; SSE2-NEXT: andl $-65536, %r15d # imm = 0xFFFF0000 +; SSE2-NEXT: movd %r15d, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: movzwl %ax, %r12d -; SSE2-NEXT: orl %ebx, %r12d +; SSE2-NEXT: pextrw $0, %xmm0, %r15d +; SSE2-NEXT: shll $16, %r15d +; SSE2-NEXT: addl {{[-0-9]+}}(%r{{[sb]}}p), %r15d # 4-byte Folded Reload ; SSE2-NEXT: shll $16, %r13d ; SSE2-NEXT: movd %r13d, %xmm1 -; SSE2-NEXT: shll $16, %r15d -; SSE2-NEXT: movd %r15d, %xmm0 +; SSE2-NEXT: shll $16, %r12d +; SSE2-NEXT: movd %r12d, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %r14d -; SSE2-NEXT: shll $16, %r14d +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movd %ebp, %xmm1 -; SSE2-NEXT: movq (%rsp), %rax # 8-byte Reload -; SSE2-NEXT: shll $16, %eax -; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: shll $16, %ebx +; SSE2-NEXT: movd %ebx, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: movzwl %ax, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx +; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: orl %r14d, %ebx ; SSE2-NEXT: shlq $32, %rbx -; SSE2-NEXT: orq %r12, %rbx +; SSE2-NEXT: orq %r15, %rbx ; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload ; SSE2-NEXT: movl %r15d, %eax -; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000 +; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Reload ; SSE2-NEXT: movl %r14d, %eax -; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000 +; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp -; SSE2-NEXT: shll $16, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: movzwl %ax, %ebp ; SSE2-NEXT: movq %r15, %rax -; SSE2-NEXT: shll $16, %eax +; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000 ; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: movq %r14, %rax -; SSE2-NEXT: shll $16, %eax +; SSE2-NEXT: andl $-65536, %eax # imm = 0xFFFF0000 ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: movzwl %ax, %r14d +; SSE2-NEXT: pextrw $0, %xmm0, %r14d +; SSE2-NEXT: shll $16, %r14d ; SSE2-NEXT: orl %ebp, %r14d ; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload ; SSE2-NEXT: shll $16, %eax @@ -728,8 +792,8 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp -; SSE2-NEXT: shll $16, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: movzwl %ax, %ebp ; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload ; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: movd %eax, %xmm1 @@ -738,8 +802,8 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; SSE2-NEXT: movd %eax, %xmm0 ; SSE2-NEXT: addss %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: movzwl %ax, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: shll $16, %eax ; SSE2-NEXT: orl %ebp, %eax ; SSE2-NEXT: shlq $32, %rax ; SSE2-NEXT: orq %r14, %rax @@ -774,7 +838,8 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm2, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill ; BF16-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload ; BF16-NEXT: vpextrw $6, %xmm0, %eax ; BF16-NEXT: shll $16, %eax @@ -785,7 +850,7 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovd %xmm0, %ebp +; BF16-NEXT: vpextrw $0, %xmm0, %ebp ; BF16-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload ; BF16-NEXT: vpextrw $5, %xmm0, %eax ; BF16-NEXT: shll $16, %eax @@ -796,7 +861,7 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovd %xmm0, %r14d +; BF16-NEXT: vpextrw $0, %xmm0, %r14d ; BF16-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload ; BF16-NEXT: vpextrw $4, %xmm0, %eax ; BF16-NEXT: shll $16, %eax @@ -807,7 +872,7 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovd %xmm0, %r15d +; BF16-NEXT: vpextrw $0, %xmm0, %r15d ; BF16-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload ; BF16-NEXT: vpextrw $3, %xmm0, %eax ; BF16-NEXT: shll $16, %eax @@ -818,7 +883,7 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovd %xmm0, %r12d +; BF16-NEXT: vpextrw $0, %xmm0, %r12d ; BF16-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload ; BF16-NEXT: vpextrw $2, %xmm0, %eax ; BF16-NEXT: shll $16, %eax @@ -829,31 +894,31 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovd %xmm0, %r13d +; BF16-NEXT: vpextrw $0, %xmm0, %r13d ; BF16-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; BF16-NEXT: vpextrw $1, %xmm0, %eax +; BF16-NEXT: vmovd %xmm0, %eax ; BF16-NEXT: shll $16, %eax ; BF16-NEXT: vmovd %eax, %xmm0 ; BF16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; BF16-NEXT: vpextrw $1, %xmm1, %eax +; BF16-NEXT: vmovd %xmm1, %eax ; BF16-NEXT: shll $16, %eax ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovd %xmm0, %ebx +; BF16-NEXT: vpextrw $0, %xmm0, %ebx ; BF16-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; BF16-NEXT: vmovd %xmm0, %eax +; BF16-NEXT: vpextrw $1, %xmm0, %eax ; BF16-NEXT: shll $16, %eax ; BF16-NEXT: vmovd %eax, %xmm0 ; BF16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; BF16-NEXT: vmovd %xmm1, %eax +; BF16-NEXT: vpextrw $1, %xmm1, %eax ; BF16-NEXT: shll $16, %eax ; BF16-NEXT: vmovd %eax, %xmm1 ; BF16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; BF16-NEXT: callq __truncsfbf2@PLT -; BF16-NEXT: vmovd %xmm0, %eax -; BF16-NEXT: vmovd %eax, %xmm0 -; BF16-NEXT: vpinsrw $1, %ebx, %xmm0, %xmm0 +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: vmovd %ebx, %xmm0 +; BF16-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 ; BF16-NEXT: vpinsrw $2, %r13d, %xmm0, %xmm0 ; BF16-NEXT: vpinsrw $3, %r12d, %xmm0, %xmm0 ; BF16-NEXT: vpinsrw $4, %r15d, %xmm0, %xmm0 @@ -871,13 +936,7 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; ; FP16-LABEL: addv: ; FP16: # %bb.0: -; FP16-NEXT: pushq %rbp -; FP16-NEXT: pushq %r15 -; FP16-NEXT: pushq %r14 -; FP16-NEXT: pushq %r13 -; FP16-NEXT: pushq %r12 -; FP16-NEXT: pushq %rbx -; FP16-NEXT: subq $40, %rsp +; FP16-NEXT: subq $152, %rsp ; FP16-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill ; FP16-NEXT: vmovw %xmm1, %eax @@ -888,40 +947,40 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm2, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; FP16-NEXT: vpextrw $7, %xmm0, %eax +; FP16-NEXT: vpextrw $1, %xmm0, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm0 ; FP16-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; FP16-NEXT: vpextrw $7, %xmm1, %eax +; FP16-NEXT: vpextrw $1, %xmm1, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovd %xmm0, %ebp +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; FP16-NEXT: vpextrw $6, %xmm0, %eax +; FP16-NEXT: vpextrw $2, %xmm0, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm0 ; FP16-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; FP16-NEXT: vpextrw $6, %xmm1, %eax +; FP16-NEXT: vpextrw $2, %xmm1, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovd %xmm0, %r14d +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; FP16-NEXT: vpextrw $5, %xmm0, %eax +; FP16-NEXT: vpextrw $3, %xmm0, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm0 ; FP16-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; FP16-NEXT: vpextrw $5, %xmm1, %eax +; FP16-NEXT: vpextrw $3, %xmm1, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovd %xmm0, %r15d +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; FP16-NEXT: vpextrw $4, %xmm0, %eax ; FP16-NEXT: shll $16, %eax @@ -932,171 +991,55 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovd %xmm0, %r12d +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; FP16-NEXT: vpextrw $3, %xmm0, %eax +; FP16-NEXT: vpextrw $5, %xmm0, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm0 ; FP16-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; FP16-NEXT: vpextrw $3, %xmm1, %eax +; FP16-NEXT: vpextrw $5, %xmm1, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovd %xmm0, %r13d +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; FP16-NEXT: vpextrw $2, %xmm0, %eax +; FP16-NEXT: vpextrw $6, %xmm0, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm0 ; FP16-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; FP16-NEXT: vpextrw $2, %xmm1, %eax +; FP16-NEXT: vpextrw $6, %xmm1, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovd %xmm0, %ebx +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; FP16-NEXT: vpextrw $1, %xmm0, %eax +; FP16-NEXT: vpextrw $7, %xmm0, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm0 ; FP16-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; FP16-NEXT: vpextrw $1, %xmm1, %eax +; FP16-NEXT: vpextrw $7, %xmm1, %eax ; FP16-NEXT: shll $16, %eax ; FP16-NEXT: vmovd %eax, %xmm1 ; FP16-NEXT: vaddss %xmm0, %xmm1, %xmm0 ; FP16-NEXT: callq __truncsfbf2@PLT -; FP16-NEXT: vmovd %xmm0, %eax -; FP16-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; FP16-NEXT: # xmm0 = mem[0],zero,zero,zero -; FP16-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 -; FP16-NEXT: vpinsrw $2, %ebx, %xmm0, %xmm0 -; FP16-NEXT: vpinsrw $3, %r13d, %xmm0, %xmm0 -; FP16-NEXT: vpinsrw $4, %r12d, %xmm0, %xmm0 -; FP16-NEXT: vpinsrw $5, %r15d, %xmm0, %xmm0 -; FP16-NEXT: vpinsrw $6, %r14d, %xmm0, %xmm0 -; FP16-NEXT: vpinsrw $7, %ebp, %xmm0, %xmm0 -; FP16-NEXT: addq $40, %rsp -; FP16-NEXT: popq %rbx -; FP16-NEXT: popq %r12 -; FP16-NEXT: popq %r13 -; FP16-NEXT: popq %r14 -; FP16-NEXT: popq %r15 -; FP16-NEXT: popq %rbp +; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; FP16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; FP16-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; FP16-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; FP16-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; FP16-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; FP16-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; FP16-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload +; FP16-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3] +; FP16-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; FP16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; FP16-NEXT: addq $152, %rsp ; FP16-NEXT: retq -; -; AVXNC-LABEL: addv: -; AVXNC: # %bb.0: -; AVXNC-NEXT: pushq %rbp -; AVXNC-NEXT: pushq %r15 -; AVXNC-NEXT: pushq %r14 -; AVXNC-NEXT: pushq %r13 -; AVXNC-NEXT: pushq %r12 -; AVXNC-NEXT: pushq %rbx -; AVXNC-NEXT: subq $40, %rsp -; AVXNC-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill -; AVXNC-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVXNC-NEXT: vpextrw $7, %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm2 -; AVXNC-NEXT: vpextrw $7, %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm2, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; AVXNC-NEXT: vpextrw $6, %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVXNC-NEXT: vpextrw $6, %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %ebp -; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; AVXNC-NEXT: vpextrw $5, %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVXNC-NEXT: vpextrw $5, %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %r14d -; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; AVXNC-NEXT: vpextrw $4, %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVXNC-NEXT: vpextrw $4, %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %r15d -; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; AVXNC-NEXT: vpextrw $3, %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVXNC-NEXT: vpextrw $3, %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %r12d -; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; AVXNC-NEXT: vpextrw $2, %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVXNC-NEXT: vpextrw $2, %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %r13d -; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; AVXNC-NEXT: vpextrw $1, %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVXNC-NEXT: vpextrw $1, %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %ebx -; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload -; AVXNC-NEXT: vmovd %xmm0, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVXNC-NEXT: vmovd %xmm1, %eax -; AVXNC-NEXT: shll $16, %eax -; AVXNC-NEXT: vmovd %eax, %xmm1 -; AVXNC-NEXT: vaddss %xmm0, %xmm1, %xmm0 -; AVXNC-NEXT: callq __truncsfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vpinsrw $1, %ebx, %xmm0, %xmm0 -; AVXNC-NEXT: vpinsrw $2, %r13d, %xmm0, %xmm0 -; AVXNC-NEXT: vpinsrw $3, %r12d, %xmm0, %xmm0 -; AVXNC-NEXT: vpinsrw $4, %r15d, %xmm0, %xmm0 -; AVXNC-NEXT: vpinsrw $5, %r14d, %xmm0, %xmm0 -; AVXNC-NEXT: vpinsrw $6, %ebp, %xmm0, %xmm0 -; AVXNC-NEXT: vpinsrw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: addq $40, %rsp -; AVXNC-NEXT: popq %rbx -; AVXNC-NEXT: popq %r12 -; AVXNC-NEXT: popq %r13 -; AVXNC-NEXT: popq %r14 -; AVXNC-NEXT: popq %r15 -; AVXNC-NEXT: popq %rbp -; AVXNC-NEXT: retq %add = fadd <8 x bfloat> %a, %b ret <8 x bfloat> %add } @@ -1104,26 +1047,28 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind { define <2 x bfloat> @pr62997(bfloat %a, bfloat %b) { ; X86-LABEL: pr62997: ; X86: # %bb.0: -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $1, {{[0-9]+}}(%esp), %xmm0, %xmm0 +; X86-NEXT: vmovsh {{[0-9]+}}(%esp), %xmm0 +; X86-NEXT: vmovsh {{[0-9]+}}(%esp), %xmm1 +; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; X86-NEXT: retl ; ; SSE2-LABEL: pr62997: ; SSE2: # %bb.0: -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: movd %xmm1, %ecx -; SSE2-NEXT: pinsrw $0, %ecx, %xmm1 -; SSE2-NEXT: pinsrw $0, %eax, %xmm0 ; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; SSE2-NEXT: retq ; -; AVX-LABEL: pr62997: -; AVX: # %bb.0: -; AVX-NEXT: vmovd %xmm1, %eax -; AVX-NEXT: vmovd %xmm0, %ecx -; AVX-NEXT: vmovd %ecx, %xmm0 -; AVX-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 -; AVX-NEXT: retq +; BF16-LABEL: pr62997: +; BF16: # %bb.0: +; BF16-NEXT: vpextrw $0, %xmm0, %eax +; BF16-NEXT: vpextrw $0, %xmm1, %ecx +; BF16-NEXT: vmovd %eax, %xmm0 +; BF16-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0 +; BF16-NEXT: retq +; +; FP16-LABEL: pr62997: +; FP16: # %bb.0: +; FP16-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; FP16-NEXT: retq %1 = insertelement <2 x bfloat> undef, bfloat %a, i64 0 %2 = insertelement <2 x bfloat> %1, bfloat %b, i64 1 ret <2 x bfloat> %2 @@ -1159,7 +1104,7 @@ define <32 x bfloat> @pr63017() { define <32 x bfloat> @pr63017_2() nounwind { ; X86-LABEL: pr63017_2: ; X86: # %bb.0: -; X86-NEXT: vpbroadcastw {{.*#+}} zmm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024] +; X86-NEXT: vpbroadcastw {{.*#+}} zmm0 = [-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0] ; X86-NEXT: vmovdqu16 (%eax), %zmm0 {%k1} ; X86-NEXT: retl ; @@ -1492,8 +1437,8 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill ; SSE2-NEXT: movd %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill ; SSE2-NEXT: movd %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill -; SSE2-NEXT: movd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill ; SSE2-NEXT: movd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill +; SSE2-NEXT: movd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill ; SSE2-NEXT: movd %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill ; SSE2-NEXT: movd %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill ; SSE2-NEXT: movd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill @@ -1509,23 +1454,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill ; SSE2-NEXT: .LBB12_67: # %else92 ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1535,23 +1480,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1563,23 +1508,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1589,23 +1534,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1617,23 +1562,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1643,23 +1588,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1671,23 +1616,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1697,23 +1642,23 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload ; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebx, %eax ; SSE2-NEXT: shlq $32, %rax @@ -1729,11 +1674,11 @@ define <32 x bfloat> @pr63017_2() nounwind { ; SSE2-NEXT: popq %r14 ; SSE2-NEXT: retq ; -; F16-LABEL: pr63017_2: -; F16: # %bb.0: -; F16-NEXT: vpbroadcastw {{.*#+}} zmm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024] -; F16-NEXT: vmovdqu16 (%rax), %zmm0 {%k1} -; F16-NEXT: retq +; FP16-LABEL: pr63017_2: +; FP16: # %bb.0: +; FP16-NEXT: vpbroadcastw {{.*#+}} zmm0 = [-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0,-1.875E+0] +; FP16-NEXT: vmovdqu16 (%rax), %zmm0 {%k1} +; FP16-NEXT: retq ; ; AVXNC-LABEL: pr63017_2: ; AVXNC: # %bb.0: @@ -1999,7 +1944,7 @@ define <32 x bfloat> @pr62997_3(<32 x bfloat> %0, bfloat %1) { ; SSE2-NEXT: movabsq $-4294967296, %rcx # imm = 0xFFFFFFFF00000000 ; SSE2-NEXT: andq %rax, %rcx ; SSE2-NEXT: movzwl %ax, %eax -; SSE2-NEXT: movd %xmm4, %edx +; SSE2-NEXT: pextrw $0, %xmm4, %edx ; SSE2-NEXT: shll $16, %edx ; SSE2-NEXT: orl %eax, %edx ; SSE2-NEXT: orq %rcx, %rdx @@ -2007,16 +1952,16 @@ define <32 x bfloat> @pr62997_3(<32 x bfloat> %0, bfloat %1) { ; SSE2-NEXT: movsd {{.*#+}} xmm0 = xmm4[0],xmm0[1] ; SSE2-NEXT: retq ; -; F16-LABEL: pr62997_3: -; F16: # %bb.0: -; F16-NEXT: vmovd %xmm1, %eax -; F16-NEXT: vpinsrw $1, %eax, %xmm0, %xmm1 -; F16-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 -; F16-NEXT: retq +; FP16-LABEL: pr62997_3: +; FP16: # %bb.0: +; FP16-NEXT: vmovw %xmm1, %eax +; FP16-NEXT: vpinsrw $1, %eax, %xmm0, %xmm1 +; FP16-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm0 +; FP16-NEXT: retq ; ; AVXNC-LABEL: pr62997_3: ; AVXNC: # %bb.0: -; AVXNC-NEXT: vmovd %xmm2, %eax +; AVXNC-NEXT: vpextrw $0, %xmm2, %eax ; AVXNC-NEXT: vpinsrw $1, %eax, %xmm0, %xmm2 ; AVXNC-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVXNC-NEXT: retq @@ -2334,42 +2279,28 @@ define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind { ; ; SSE2-LABEL: fptrunc_v4f32: ; SSE2: # %bb.0: -; SSE2-NEXT: pushq %rbp -; SSE2-NEXT: pushq %r14 -; SSE2-NEXT: pushq %rbx -; SSE2-NEXT: subq $32, %rsp -; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE2-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1] +; SSE2-NEXT: subq $72, %rsp +; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload +; SSE2-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx -; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero -; SSE2-NEXT: movd %xmm0, %ebp -; SSE2-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; SSE2-NEXT: # xmm0 = mem[0],zero,zero,zero -; SSE2-NEXT: movd %xmm0, %r14d -; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax -; SSE2-NEXT: pinsrw $0, %eax, %xmm0 -; SSE2-NEXT: pinsrw $0, %r14d, %xmm1 +; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE2-NEXT: pinsrw $0, %ebp, %xmm0 -; SSE2-NEXT: pinsrw $0, %ebx, %xmm2 -; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE2-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; SSE2-NEXT: addq $32, %rsp -; SSE2-NEXT: popq %rbx -; SSE2-NEXT: popq %r14 -; SSE2-NEXT: popq %rbp +; SSE2-NEXT: addq $72, %rsp ; SSE2-NEXT: retq ; ; F16-LABEL: fptrunc_v4f32: @@ -2406,22 +2337,22 @@ define <8 x bfloat> @fptrunc_v8f32(<8 x float> %a) nounwind { ; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %ebx ; SSE2-NEXT: orl %ebp, %ebx ; SSE2-NEXT: shlq $32, %rbx @@ -2429,22 +2360,22 @@ define <8 x bfloat> @fptrunc_v8f32(<8 x float> %a) nounwind { ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebp, %r14d ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebp, %eax ; SSE2-NEXT: shlq $32, %rax @@ -2494,22 +2425,22 @@ define <16 x bfloat> @fptrunc_v16f32(<16 x float> %a) nounwind { ; SSE2-NEXT: movaps %xmm2, %xmm0 ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %ebx ; SSE2-NEXT: orl %ebp, %ebx ; SSE2-NEXT: shlq $32, %rbx @@ -2517,22 +2448,22 @@ define <16 x bfloat> @fptrunc_v16f32(<16 x float> %a) nounwind { ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r15d ; SSE2-NEXT: orl %ebp, %r15d ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebp, %r14d ; SSE2-NEXT: shlq $32, %r14 @@ -2540,22 +2471,22 @@ define <16 x bfloat> @fptrunc_v16f32(<16 x float> %a) nounwind { ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r12d ; SSE2-NEXT: orl %ebp, %r12d ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r15d ; SSE2-NEXT: orl %ebp, %r15d ; SSE2-NEXT: shlq $32, %r15 @@ -2563,22 +2494,22 @@ define <16 x bfloat> @fptrunc_v16f32(<16 x float> %a) nounwind { ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r12d ; SSE2-NEXT: orl %ebp, %r12d ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncsfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebp, %eax ; SSE2-NEXT: shlq $32, %rax @@ -2616,87 +2547,64 @@ define <16 x bfloat> @fptrunc_v16f32(<16 x float> %a) nounwind { define <8 x bfloat> @fptrunc_v8f64(<8 x double> %a) nounwind { ; X86-LABEL: fptrunc_v8f64: ; X86: # %bb.0: -; X86-NEXT: subl $220, %esp +; X86-NEXT: subl $204, %esp ; X86-NEXT: vmovups %zmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 64-byte Spill -; X86-NEXT: vmovhps %xmm0, (%esp) -; X86-NEXT: vzeroupper -; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill -; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 # 64-byte Reload ; X86-NEXT: vextractf128 $1, %ymm0, %xmm0 ; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovlps %xmm0, (%esp) ; X86-NEXT: vzeroupper ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload ; X86-NEXT: vmovhps %xmm0, (%esp) ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill -; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 # 64-byte Reload -; X86-NEXT: vextractf32x4 $2, %zmm0, %xmm0 ; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill +; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 # 64-byte Reload ; X86-NEXT: vmovlps %xmm0, (%esp) ; X86-NEXT: vzeroupper ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill -; X86-NEXT: vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill +; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 # 64-byte Reload ; X86-NEXT: vmovhps %xmm0, (%esp) +; X86-NEXT: vzeroupper ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 # 64-byte Reload -; X86-NEXT: vextractf32x4 $3, %zmm0, %xmm0 +; X86-NEXT: vextractf32x4 $2, %zmm0, %xmm0 ; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovlps %xmm0, (%esp) ; X86-NEXT: vzeroupper ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload ; X86-NEXT: vmovhps %xmm0, (%esp) ; X86-NEXT: calll __truncdfbf2 +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovups {{[-0-9]+}}(%e{{[sb]}}p), %zmm0 # 64-byte Reload +; X86-NEXT: vextractf32x4 $3, %zmm0, %xmm0 +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vmovlps %xmm0, (%esp) -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload -; X86-NEXT: fstps {{[0-9]+}}(%esp) ; X86-NEXT: vzeroupper ; X86-NEXT: calll __truncdfbf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero -; X86-NEXT: vmovd %xmm1, %eax -; X86-NEXT: vmovd %eax, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $1, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $4, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1 -; X86-NEXT: vmovd %xmm0, %eax -; X86-NEXT: vpinsrw $7, %eax, %xmm1, %xmm0 -; X86-NEXT: addl $220, %esp +; X86-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill +; X86-NEXT: vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload +; X86-NEXT: vmovhps %xmm0, (%esp) +; X86-NEXT: calll __truncdfbf2 +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; X86-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; X86-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload +; X86-NEXT: vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload +; X86-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3] +; X86-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; X86-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; X86-NEXT: addl $204, %esp ; X86-NEXT: retl ; ; SSE2-LABEL: fptrunc_v8f64: @@ -2711,21 +2619,21 @@ define <8 x bfloat> @fptrunc_v8f64(<8 x double> %a) nounwind { ; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebx +; SSE2-NEXT: pextrw $0, %xmm0, %ebx ; SSE2-NEXT: shll $16, %ebx ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebx, %r14d ; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %ebx ; SSE2-NEXT: orl %ebp, %ebx ; SSE2-NEXT: shlq $32, %rbx @@ -2733,21 +2641,21 @@ define <8 x bfloat> @fptrunc_v8f64(<8 x double> %a) nounwind { ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %r14d ; SSE2-NEXT: orl %ebp, %r14d ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %ebp +; SSE2-NEXT: pextrw $0, %xmm0, %ebp ; SSE2-NEXT: shll $16, %ebp ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __truncdfbf2@PLT -; SSE2-NEXT: movd %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %eax ; SSE2-NEXT: movzwl %ax, %eax ; SSE2-NEXT: orl %ebp, %eax ; SSE2-NEXT: shlq $32, %rax @@ -2761,89 +2669,63 @@ define <8 x bfloat> @fptrunc_v8f64(<8 x double> %a) nounwind { ; SSE2-NEXT: popq %rbp ; SSE2-NEXT: retq ; -; F16-LABEL: fptrunc_v8f64: -; F16: # %bb.0: -; F16-NEXT: pushq %rbp -; F16-NEXT: pushq %r15 -; F16-NEXT: pushq %r14 -; F16-NEXT: pushq %r13 -; F16-NEXT: pushq %r12 -; F16-NEXT: pushq %rbx -; F16-NEXT: subq $136, %rsp -; F16-NEXT: vmovupd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill -; F16-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0] -; F16-NEXT: vzeroupper -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; F16-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; F16-NEXT: vextractf128 $1, %ymm0, %xmm0 -; F16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; F16-NEXT: vzeroupper -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; F16-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; F16-NEXT: # xmm0 = mem[1,0] -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; F16-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; F16-NEXT: vextractf32x4 $2, %zmm0, %xmm0 -; F16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; F16-NEXT: vzeroupper -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; F16-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; F16-NEXT: # xmm0 = mem[1,0] -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; F16-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; F16-NEXT: vextractf32x4 $3, %zmm0, %xmm0 -; F16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; F16-NEXT: vzeroupper -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; F16-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; F16-NEXT: # xmm0 = mem[1,0] -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; F16-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; F16-NEXT: # xmm0 = mem[0],zero,zero,zero -; F16-NEXT: vmovd %xmm0, %ebp -; F16-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; F16-NEXT: # xmm0 = mem[0],zero,zero,zero -; F16-NEXT: vmovd %xmm0, %r14d -; F16-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; F16-NEXT: # xmm0 = mem[0],zero,zero,zero -; F16-NEXT: vmovd %xmm0, %r15d -; F16-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; F16-NEXT: # xmm0 = mem[0],zero,zero,zero -; F16-NEXT: vmovd %xmm0, %r12d -; F16-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; F16-NEXT: # xmm0 = mem[0],zero,zero,zero -; F16-NEXT: vmovd %xmm0, %r13d -; F16-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; F16-NEXT: # xmm0 = mem[0],zero,zero,zero -; F16-NEXT: vmovd %xmm0, %ebx -; F16-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; F16-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 -; F16-NEXT: vzeroupper -; F16-NEXT: callq __truncdfbf2@PLT -; F16-NEXT: vmovd %xmm0, %eax -; F16-NEXT: vmovd %eax, %xmm0 -; F16-NEXT: vpinsrw $1, %ebx, %xmm0, %xmm0 -; F16-NEXT: vpinsrw $2, %r13d, %xmm0, %xmm0 -; F16-NEXT: vpinsrw $3, %r12d, %xmm0, %xmm0 -; F16-NEXT: vpinsrw $4, %r15d, %xmm0, %xmm0 -; F16-NEXT: vpinsrw $5, %r14d, %xmm0, %xmm0 -; F16-NEXT: vpinsrw $6, %ebp, %xmm0, %xmm0 -; F16-NEXT: vpinsrw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload -; F16-NEXT: addq $136, %rsp -; F16-NEXT: popq %rbx -; F16-NEXT: popq %r12 -; F16-NEXT: popq %r13 -; F16-NEXT: popq %r14 -; F16-NEXT: popq %r15 -; F16-NEXT: popq %rbp -; F16-NEXT: retq +; FP16-LABEL: fptrunc_v8f64: +; FP16: # %bb.0: +; FP16-NEXT: subq $184, %rsp +; FP16-NEXT: vmovupd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill +; FP16-NEXT: vextractf128 $1, %ymm0, %xmm0 +; FP16-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0] +; FP16-NEXT: vzeroupper +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; FP16-NEXT: # xmm0 = mem[1,0] +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload +; FP16-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; FP16-NEXT: vzeroupper +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload +; FP16-NEXT: vextractf32x4 $2, %zmm0, %xmm0 +; FP16-NEXT: vmovapd %xmm0, (%rsp) # 16-byte Spill +; FP16-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0] +; FP16-NEXT: vzeroupper +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; FP16-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload +; FP16-NEXT: vextractf32x4 $3, %zmm0, %xmm0 +; FP16-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0] +; FP16-NEXT: vzeroupper +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; FP16-NEXT: callq __truncdfbf2@PLT +; FP16-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; FP16-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] +; FP16-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload +; FP16-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; FP16-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; FP16-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; FP16-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; FP16-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] +; FP16-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; FP16-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload +; FP16-NEXT: # xmm2 = xmm2[0],mem[0],xmm2[1],mem[1],xmm2[2],mem[2],xmm2[3],mem[3] +; FP16-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; FP16-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; FP16-NEXT: addq $184, %rsp +; FP16-NEXT: retq ; ; AVXNC-LABEL: fptrunc_v8f64: ; AVXNC: # %bb.0: @@ -2853,74 +2735,68 @@ define <8 x bfloat> @fptrunc_v8f64(<8 x double> %a) nounwind { ; AVXNC-NEXT: pushq %r13 ; AVXNC-NEXT: pushq %r12 ; AVXNC-NEXT: pushq %rbx -; AVXNC-NEXT: subq $120, %rsp +; AVXNC-NEXT: subq $168, %rsp ; AVXNC-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVXNC-NEXT: vmovupd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVXNC-NEXT: vshufpd {{.*#+}} xmm0 = xmm0[1,0] +; AVXNC-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVXNC-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 ; AVXNC-NEXT: vzeroupper ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; AVXNC-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVXNC-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVXNC-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVXNC-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVXNC-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill ; AVXNC-NEXT: vzeroupper ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; AVXNC-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; AVXNC-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVXNC-NEXT: vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload ; AVXNC-NEXT: # xmm0 = mem[1,0] ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; AVXNC-NEXT: vmovapd %xmm0, (%rsp) # 16-byte Spill ; AVXNC-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVXNC-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 ; AVXNC-NEXT: vzeroupper ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; AVXNC-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVXNC-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload ; AVXNC-NEXT: # xmm0 = mem[1,0] ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; AVXNC-NEXT: vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVXNC-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVXNC-NEXT: vextractf128 $1, %ymm0, %xmm0 ; AVXNC-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVXNC-NEXT: vzeroupper ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; AVXNC-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVXNC-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload ; AVXNC-NEXT: # xmm0 = mem[1,0] ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill -; AVXNC-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: # xmm0 = mem[0],zero,zero,zero -; AVXNC-NEXT: vmovd %xmm0, %ebp -; AVXNC-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: # xmm0 = mem[0],zero,zero,zero -; AVXNC-NEXT: vmovd %xmm0, %r14d -; AVXNC-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: # xmm0 = mem[0],zero,zero,zero -; AVXNC-NEXT: vmovd %xmm0, %r15d -; AVXNC-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: # xmm0 = mem[0],zero,zero,zero -; AVXNC-NEXT: vmovd %xmm0, %r12d -; AVXNC-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: # xmm0 = mem[0],zero,zero,zero -; AVXNC-NEXT: vmovd %xmm0, %r13d -; AVXNC-NEXT: vmovd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: # xmm0 = mem[0],zero,zero,zero -; AVXNC-NEXT: vmovd %xmm0, %ebx -; AVXNC-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVXNC-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 -; AVXNC-NEXT: vzeroupper +; AVXNC-NEXT: vpextrw $0, %xmm0, %eax +; AVXNC-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill +; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVXNC-NEXT: vpextrw $0, %xmm0, %ebp +; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVXNC-NEXT: vpextrw $0, %xmm0, %r14d +; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVXNC-NEXT: vpextrw $0, %xmm0, %r15d +; AVXNC-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload +; AVXNC-NEXT: vpextrw $0, %xmm0, %r12d +; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVXNC-NEXT: vpextrw $0, %xmm0, %r13d +; AVXNC-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVXNC-NEXT: vpextrw $0, %xmm0, %ebx +; AVXNC-NEXT: vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; AVXNC-NEXT: # xmm0 = mem[1,0] ; AVXNC-NEXT: callq __truncdfbf2@PLT -; AVXNC-NEXT: vmovd %xmm0, %eax -; AVXNC-NEXT: vmovd %eax, %xmm0 -; AVXNC-NEXT: vpinsrw $1, %ebx, %xmm0, %xmm0 +; AVXNC-NEXT: vpextrw $0, %xmm0, %eax +; AVXNC-NEXT: vmovd %ebx, %xmm0 +; AVXNC-NEXT: vpinsrw $1, %eax, %xmm0, %xmm0 ; AVXNC-NEXT: vpinsrw $2, %r13d, %xmm0, %xmm0 ; AVXNC-NEXT: vpinsrw $3, %r12d, %xmm0, %xmm0 ; AVXNC-NEXT: vpinsrw $4, %r15d, %xmm0, %xmm0 ; AVXNC-NEXT: vpinsrw $5, %r14d, %xmm0, %xmm0 ; AVXNC-NEXT: vpinsrw $6, %ebp, %xmm0, %xmm0 ; AVXNC-NEXT: vpinsrw $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload -; AVXNC-NEXT: addq $120, %rsp +; AVXNC-NEXT: addq $168, %rsp ; AVXNC-NEXT: popq %rbx ; AVXNC-NEXT: popq %r12 ; AVXNC-NEXT: popq %r13 -- GitLab From f5f66e26b5f010218651cab831d5651fe7a71a0a Mon Sep 17 00:00:00 2001 From: XinWang10 <108658776+XinWang10@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:56:15 +0800 Subject: [PATCH 073/728] [X86]Support lowering for APX Promoted SHA/MOVDIR/CRC32/INVPCID/CET instructions (#76786) R16-R31 was added into GPRs in https://github.com/llvm/llvm-project/pull/70958, This patch supports the lowering for promoted SHA/MOVDIR/CRC32/INVPCID/CET. RFC: https://discourse.llvm.org/t/rfc-design-for-apx-feature-egpr-and-ndd-support/73031/4 --- llvm/lib/Target/X86/X86DomainReassignment.cpp | 67 ++--- llvm/lib/Target/X86/X86FastISel.cpp | 10 +- llvm/lib/Target/X86/X86InstrSystem.td | 13 +- .../X86/crc32-intrinsics-fast-isel-x86.ll | 41 ++- .../X86/crc32-intrinsics-fast-isel-x86_64.ll | 29 +- llvm/test/CodeGen/X86/crc32-intrinsics-x86.ll | 19 ++ .../CodeGen/X86/crc32-intrinsics-x86_64.ll | 13 + llvm/test/CodeGen/X86/invpcid-intrinsic.ll | 27 +- llvm/test/CodeGen/X86/movdir-intrinsic-x86.ll | 33 ++- .../CodeGen/X86/movdir-intrinsic-x86_64.ll | 12 +- llvm/test/CodeGen/X86/sha.ll | 269 ++++++++++++++---- llvm/test/CodeGen/X86/x64-cet-intrinsics.ll | 120 ++++++-- 12 files changed, 478 insertions(+), 175 deletions(-) diff --git a/llvm/lib/Target/X86/X86DomainReassignment.cpp b/llvm/lib/Target/X86/X86DomainReassignment.cpp index bdd86e48fa54..20dbaf797e32 100644 --- a/llvm/lib/Target/X86/X86DomainReassignment.cpp +++ b/llvm/lib/Target/X86/X86DomainReassignment.cpp @@ -619,40 +619,30 @@ void X86DomainReassignment::initConverters() { std::make_unique(From, To); }; - bool HasEGPR = STI->hasEGPR(); - createReplacerDstCOPY(X86::MOVZX32rm16, - HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm); - createReplacerDstCOPY(X86::MOVZX64rm16, - HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm); +#define GET_EGPR_IF_ENABLED(OPC) STI->hasEGPR() ? OPC##_EVEX : OPC + createReplacerDstCOPY(X86::MOVZX32rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm)); + createReplacerDstCOPY(X86::MOVZX64rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm)); - createReplacerDstCOPY(X86::MOVZX32rr16, - HasEGPR ? X86::KMOVWkk_EVEX : X86::KMOVWkk); - createReplacerDstCOPY(X86::MOVZX64rr16, - HasEGPR ? X86::KMOVWkk_EVEX : X86::KMOVWkk); + createReplacerDstCOPY(X86::MOVZX32rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk)); + createReplacerDstCOPY(X86::MOVZX64rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk)); if (STI->hasDQI()) { - createReplacerDstCOPY(X86::MOVZX16rm8, - HasEGPR ? X86::KMOVBkm_EVEX : X86::KMOVBkm); - createReplacerDstCOPY(X86::MOVZX32rm8, - HasEGPR ? X86::KMOVBkm_EVEX : X86::KMOVBkm); - createReplacerDstCOPY(X86::MOVZX64rm8, - HasEGPR ? X86::KMOVBkm_EVEX : X86::KMOVBkm); - - createReplacerDstCOPY(X86::MOVZX16rr8, - HasEGPR ? X86::KMOVBkk_EVEX : X86::KMOVBkk); - createReplacerDstCOPY(X86::MOVZX32rr8, - HasEGPR ? X86::KMOVBkk_EVEX : X86::KMOVBkk); - createReplacerDstCOPY(X86::MOVZX64rr8, - HasEGPR ? X86::KMOVBkk_EVEX : X86::KMOVBkk); + createReplacerDstCOPY(X86::MOVZX16rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm)); + createReplacerDstCOPY(X86::MOVZX32rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm)); + createReplacerDstCOPY(X86::MOVZX64rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm)); + + createReplacerDstCOPY(X86::MOVZX16rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk)); + createReplacerDstCOPY(X86::MOVZX32rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk)); + createReplacerDstCOPY(X86::MOVZX64rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk)); } auto createReplacer = [&](unsigned From, unsigned To) { Converters[{MaskDomain, From}] = std::make_unique(From, To); }; - createReplacer(X86::MOV16rm, HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm); - createReplacer(X86::MOV16mr, HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk); - createReplacer(X86::MOV16rr, HasEGPR ? X86::KMOVWkk_EVEX : X86::KMOVWkk); + createReplacer(X86::MOV16rm, GET_EGPR_IF_ENABLED(X86::KMOVWkm)); + createReplacer(X86::MOV16mr, GET_EGPR_IF_ENABLED(X86::KMOVWmk)); + createReplacer(X86::MOV16rr, GET_EGPR_IF_ENABLED(X86::KMOVWkk)); createReplacer(X86::SHR16ri, X86::KSHIFTRWri); createReplacer(X86::SHL16ri, X86::KSHIFTLWri); createReplacer(X86::NOT16r, X86::KNOTWrr); @@ -661,14 +651,14 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::XOR16rr, X86::KXORWrr); if (STI->hasBWI()) { - createReplacer(X86::MOV32rm, HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm); - createReplacer(X86::MOV64rm, HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm); + createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm)); + createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm)); - createReplacer(X86::MOV32mr, HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk); - createReplacer(X86::MOV64mr, HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk); + createReplacer(X86::MOV32mr, GET_EGPR_IF_ENABLED(X86::KMOVDmk)); + createReplacer(X86::MOV64mr, GET_EGPR_IF_ENABLED(X86::KMOVQmk)); - createReplacer(X86::MOV32rr, HasEGPR ? X86::KMOVDkk_EVEX : X86::KMOVDkk); - createReplacer(X86::MOV64rr, HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk); + createReplacer(X86::MOV32rr, GET_EGPR_IF_ENABLED(X86::KMOVDkk)); + createReplacer(X86::MOV64rr, GET_EGPR_IF_ENABLED(X86::KMOVQkk)); createReplacer(X86::SHR32ri, X86::KSHIFTRDri); createReplacer(X86::SHR64ri, X86::KSHIFTRQri); @@ -696,8 +686,8 @@ void X86DomainReassignment::initConverters() { // TODO: KTEST is not a replacement for TEST due to flag differences. Need // to prove only Z flag is used. - //createReplacer(X86::TEST32rr, X86::KTESTDrr); - //createReplacer(X86::TEST64rr, X86::KTESTQrr); + // createReplacer(X86::TEST32rr, X86::KTESTDrr); + // createReplacer(X86::TEST64rr, X86::KTESTQrr); } if (STI->hasDQI()) { @@ -706,9 +696,9 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::AND8rr, X86::KANDBrr); - createReplacer(X86::MOV8rm, HasEGPR ? X86::KMOVBkm_EVEX : X86::KMOVBkm); - createReplacer(X86::MOV8mr, HasEGPR ? X86::KMOVBmk_EVEX : X86::KMOVBmk); - createReplacer(X86::MOV8rr, HasEGPR ? X86::KMOVBkk_EVEX : X86::KMOVBkk); + createReplacer(X86::MOV8rm, GET_EGPR_IF_ENABLED(X86::KMOVBkm)); + createReplacer(X86::MOV8mr, GET_EGPR_IF_ENABLED(X86::KMOVBmk)); + createReplacer(X86::MOV8rr, GET_EGPR_IF_ENABLED(X86::KMOVBkk)); createReplacer(X86::NOT8r, X86::KNOTBrr); @@ -719,11 +709,12 @@ void X86DomainReassignment::initConverters() { // TODO: KTEST is not a replacement for TEST due to flag differences. Need // to prove only Z flag is used. - //createReplacer(X86::TEST8rr, X86::KTESTBrr); - //createReplacer(X86::TEST16rr, X86::KTESTWrr); + // createReplacer(X86::TEST8rr, X86::KTESTBrr); + // createReplacer(X86::TEST16rr, X86::KTESTWrr); createReplacer(X86::XOR8rr, X86::KXORBrr); } +#undef GET_EGPR_IF_ENABLED } bool X86DomainReassignment::runOnMachineFunction(MachineFunction &MF) { diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp index c789ac82ac94..1ce1e6f6a563 100644 --- a/llvm/lib/Target/X86/X86FastISel.cpp +++ b/llvm/lib/Target/X86/X86FastISel.cpp @@ -3046,22 +3046,24 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { switch (II->getIntrinsicID()) { default: llvm_unreachable("Unexpected intrinsic."); +#define GET_EGPR_IF_ENABLED(OPC) Subtarget->hasEGPR() ? OPC##_EVEX : OPC case Intrinsic::x86_sse42_crc32_32_8: - Opc = X86::CRC32r32r8; + Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r8); RC = &X86::GR32RegClass; break; case Intrinsic::x86_sse42_crc32_32_16: - Opc = X86::CRC32r32r16; + Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r16); RC = &X86::GR32RegClass; break; case Intrinsic::x86_sse42_crc32_32_32: - Opc = X86::CRC32r32r32; + Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r32); RC = &X86::GR32RegClass; break; case Intrinsic::x86_sse42_crc32_64_64: - Opc = X86::CRC32r64r64; + Opc = GET_EGPR_IF_ENABLED(X86::CRC32r64r64); RC = &X86::GR64RegClass; break; +#undef GET_EGPR_IF_ENABLED } const Value *LHS = II->getArgOperand(0); diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td index 699e5847e63f..b1be4739617d 100644 --- a/llvm/lib/Target/X86/X86InstrSystem.td +++ b/llvm/lib/Target/X86/X86InstrSystem.td @@ -695,14 +695,14 @@ def INVPCID32 : I<0x82, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), Requires<[Not64BitMode, HasINVPCID]>; def INVPCID64 : I<0x82, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invpcid\t{$src2, $src1|$src1, $src2}", []>, T8, PD, - Requires<[In64BitMode, HasINVPCID]>; + Requires<[In64BitMode]>; def INVPCID64_EVEX : I<0xF2, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invpcid\t{$src2, $src1|$src1, $src2}", []>, - EVEX, NoCD8, T_MAP4, XS, Requires<[In64BitMode, HasINVPCID]>; + EVEX, NoCD8, T_MAP4, XS, Requires<[In64BitMode]>; } // SchedRW -let Predicates = [In64BitMode, HasINVPCID] in { +let Predicates = [HasINVPCID, NoEGPR] in { // The instruction can only use a 64 bit register as the register argument // in 64 bit mode, while the intrinsic only accepts a 32 bit argument // corresponding to it. @@ -714,6 +714,13 @@ let Predicates = [In64BitMode, HasINVPCID] in { addr:$src2)>; } +let Predicates = [HasINVPCID, HasEGPR] in { + def : Pat<(int_x86_invpcid GR32:$src1, addr:$src2), + (INVPCID64_EVEX + (SUBREG_TO_REG (i64 0), (MOV32rr GR32:$src1), sub_32bit), + addr:$src2)>; +} + //===----------------------------------------------------------------------===// // SMAP Instruction diff --git a/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86.ll b/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86.ll index 056d79f379fd..873986e99777 100644 --- a/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86.ll +++ b/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86.ll @@ -3,8 +3,9 @@ ; RUN: llc < %s -fast-isel -pass-remarks-missed=sdagisel -mtriple=i686-unknown-unknown -mattr=+crc32 2>&1 >/dev/null | FileCheck %s -check-prefix=STDERR-X86 -allow-empty ; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-unknown-unknown -mattr=-sse4.2,+crc32 | FileCheck %s -check-prefix=X86 ; RUN: llc < %s -fast-isel -fast-isel-abort=1 -mtriple=i686-unknown-unknown -mattr=+crc32 | FileCheck %s -check-prefix=X86 -; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=-sse4.2,+crc32 | FileCheck %s -check-prefix=X64 -; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=+crc32 | FileCheck %s -check-prefix=X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=-sse4.2,+crc32 --show-mc-encoding | FileCheck %s -check-prefix=X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=+crc32 --show-mc-encoding | FileCheck %s -check-prefix=X64 +; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=+crc32,+egpr --show-mc-encoding | FileCheck %s -check-prefix=EGPR ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/sse42-builtins.c @@ -21,9 +22,15 @@ define i32 @test_mm_crc32_u8(i32 %a0, i32 %a1) nounwind { ; ; X64-LABEL: test_mm_crc32_u8: ; X64: # %bb.0: -; X64-NEXT: movl %edi, %eax -; X64-NEXT: crc32b %sil, %eax -; X64-NEXT: retq +; X64-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; X64-NEXT: crc32b %sil, %eax # encoding: [0xf2,0x40,0x0f,0x38,0xf0,0xc6] +; X64-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_mm_crc32_u8: +; EGPR: # %bb.0: +; EGPR-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; EGPR-NEXT: crc32b %sil, %eax # encoding: [0x62,0xf4,0x7c,0x08,0xf0,0xc6] +; EGPR-NEXT: retq # encoding: [0xc3] %trunc = trunc i32 %a1 to i8 %res = call i32 @llvm.x86.sse42.crc32.32.8(i32 %a0, i8 %trunc) ret i32 %res @@ -41,9 +48,15 @@ define i32 @test_mm_crc32_u16(i32 %a0, i32 %a1) nounwind { ; ; X64-LABEL: test_mm_crc32_u16: ; X64: # %bb.0: -; X64-NEXT: movl %edi, %eax -; X64-NEXT: crc32w %si, %eax -; X64-NEXT: retq +; X64-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; X64-NEXT: crc32w %si, %eax # encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6] +; X64-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_mm_crc32_u16: +; EGPR: # %bb.0: +; EGPR-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; EGPR-NEXT: crc32w %si, %eax # encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6] +; EGPR-NEXT: retq # encoding: [0xc3] %trunc = trunc i32 %a1 to i16 %res = call i32 @llvm.x86.sse42.crc32.32.16(i32 %a0, i16 %trunc) ret i32 %res @@ -59,9 +72,15 @@ define i32 @test_mm_crc32_u32(i32 %a0, i32 %a1) nounwind { ; ; X64-LABEL: test_mm_crc32_u32: ; X64: # %bb.0: -; X64-NEXT: movl %edi, %eax -; X64-NEXT: crc32l %esi, %eax -; X64-NEXT: retq +; X64-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; X64-NEXT: crc32l %esi, %eax # encoding: [0xf2,0x0f,0x38,0xf1,0xc6] +; X64-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_mm_crc32_u32: +; EGPR: # %bb.0: +; EGPR-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; EGPR-NEXT: crc32l %esi, %eax # encoding: [0x62,0xf4,0x7c,0x08,0xf1,0xc6] +; EGPR-NEXT: retq # encoding: [0xc3] %res = call i32 @llvm.x86.sse42.crc32.32.32(i32 %a0, i32 %a1) ret i32 %res } diff --git a/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86_64.ll b/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86_64.ll index e0ec432b3854..71d955bda752 100644 --- a/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86_64.ll +++ b/llvm/test/CodeGen/X86/crc32-intrinsics-fast-isel-x86_64.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=-sse4.2,+crc32 | FileCheck %s -; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=+crc32 | FileCheck %s +; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=-sse4.2,+crc32 --show-mc-encoding | FileCheck %s +; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=+crc32 --show-mc-encoding | FileCheck %s +; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64-unknown-unknown -mattr=+crc32,+egpr --show-mc-encoding | FileCheck %s -check-prefix=EGPR ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/sse42-builtins.c @@ -8,9 +9,15 @@ define i64 @test_mm_crc64_u8(i64 %a0, i32 %a1) nounwind{ ; CHECK-LABEL: test_mm_crc64_u8: ; CHECK: # %bb.0: -; CHECK-NEXT: crc32b %sil, %edi -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: retq +; CHECK-NEXT: crc32b %sil, %edi # encoding: [0xf2,0x40,0x0f,0x38,0xf0,0xfe] +; CHECK-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; CHECK-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_mm_crc64_u8: +; EGPR: # %bb.0: +; EGPR-NEXT: crc32b %sil, %edi # encoding: [0x62,0xf4,0x7c,0x08,0xf0,0xfe] +; EGPR-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; EGPR-NEXT: retq # encoding: [0xc3] %trunc = trunc i32 %a1 to i8 %res = call i64 @llvm.x86.sse42.crc32.64.8(i64 %a0, i8 %trunc) ret i64 %res @@ -20,9 +27,15 @@ declare i64 @llvm.x86.sse42.crc32.64.8(i64, i8) nounwind readnone define i64 @test_mm_crc64_u64(i64 %a0, i64 %a1) nounwind{ ; CHECK-LABEL: test_mm_crc64_u64: ; CHECK: # %bb.0: -; CHECK-NEXT: movq %rdi, %rax -; CHECK-NEXT: crc32q %rsi, %rax -; CHECK-NEXT: retq +; CHECK-NEXT: movq %rdi, %rax # encoding: [0x48,0x89,0xf8] +; CHECK-NEXT: crc32q %rsi, %rax # encoding: [0xf2,0x48,0x0f,0x38,0xf1,0xc6] +; CHECK-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_mm_crc64_u64: +; EGPR: # %bb.0: +; EGPR-NEXT: movq %rdi, %rax # encoding: [0x48,0x89,0xf8] +; EGPR-NEXT: crc32q %rsi, %rax # encoding: [0x62,0xf4,0xfc,0x08,0xf1,0xc6] +; EGPR-NEXT: retq # encoding: [0xc3] %res = call i64 @llvm.x86.sse42.crc32.64.64(i64 %a0, i64 %a1) ret i64 %res } diff --git a/llvm/test/CodeGen/X86/crc32-intrinsics-x86.ll b/llvm/test/CodeGen/X86/crc32-intrinsics-x86.ll index 7623ba68353e..84c7f90cfe3c 100644 --- a/llvm/test/CodeGen/X86/crc32-intrinsics-x86.ll +++ b/llvm/test/CodeGen/X86/crc32-intrinsics-x86.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -disable-peephole -mtriple=i386-apple-darwin -mattr=+crc32 -show-mc-encoding | FileCheck %s --check-prefixes=X86 ; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+crc32 -show-mc-encoding | FileCheck %s --check-prefixes=X64 +; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+crc32,+egpr -show-mc-encoding | FileCheck %s --check-prefixes=EGPR define i32 @crc32_32_8(i32 %a, i8 %b) nounwind { ; X86-LABEL: crc32_32_8: @@ -14,6 +15,12 @@ define i32 @crc32_32_8(i32 %a, i8 %b) nounwind { ; X64-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] ; X64-NEXT: crc32b %sil, %eax ## encoding: [0xf2,0x40,0x0f,0x38,0xf0,0xc6] ; X64-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: crc32_32_8: +; EGPR: ## %bb.0: +; EGPR-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] +; EGPR-NEXT: crc32b %sil, %eax ## encoding: [0x62,0xf4,0x7c,0x08,0xf0,0xc6] +; EGPR-NEXT: retq ## encoding: [0xc3] %tmp = call i32 @llvm.x86.sse42.crc32.32.8(i32 %a, i8 %b) ret i32 %tmp } @@ -31,6 +38,12 @@ define i32 @crc32_32_16(i32 %a, i16 %b) nounwind { ; X64-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] ; X64-NEXT: crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6] ; X64-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: crc32_32_16: +; EGPR: ## %bb.0: +; EGPR-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] +; EGPR-NEXT: crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6] +; EGPR-NEXT: retq ## encoding: [0xc3] %tmp = call i32 @llvm.x86.sse42.crc32.32.16(i32 %a, i16 %b) ret i32 %tmp } @@ -48,6 +61,12 @@ define i32 @crc32_32_32(i32 %a, i32 %b) nounwind { ; X64-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] ; X64-NEXT: crc32l %esi, %eax ## encoding: [0xf2,0x0f,0x38,0xf1,0xc6] ; X64-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: crc32_32_32: +; EGPR: ## %bb.0: +; EGPR-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] +; EGPR-NEXT: crc32l %esi, %eax ## encoding: [0x62,0xf4,0x7c,0x08,0xf1,0xc6] +; EGPR-NEXT: retq ## encoding: [0xc3] %tmp = call i32 @llvm.x86.sse42.crc32.32.32(i32 %a, i32 %b) ret i32 %tmp } diff --git a/llvm/test/CodeGen/X86/crc32-intrinsics-x86_64.ll b/llvm/test/CodeGen/X86/crc32-intrinsics-x86_64.ll index b0f7a394f07b..bda26a15b277 100644 --- a/llvm/test/CodeGen/X86/crc32-intrinsics-x86_64.ll +++ b/llvm/test/CodeGen/X86/crc32-intrinsics-x86_64.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+crc32 -show-mc-encoding | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+crc32,+egpr -show-mc-encoding | FileCheck %s --check-prefixes=EGPR declare i64 @llvm.x86.sse42.crc32.64.8(i64, i8) nounwind declare i64 @llvm.x86.sse42.crc32.64.64(i64, i64) nounwind @@ -10,6 +11,12 @@ define i64 @crc32_64_8(i64 %a, i8 %b) nounwind { ; CHECK-NEXT: movq %rdi, %rax ## encoding: [0x48,0x89,0xf8] ; CHECK-NEXT: crc32b %sil, %eax ## encoding: [0xf2,0x40,0x0f,0x38,0xf0,0xc6] ; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: crc32_64_8: +; EGPR: ## %bb.0: +; EGPR-NEXT: movq %rdi, %rax ## encoding: [0x48,0x89,0xf8] +; EGPR-NEXT: crc32b %sil, %eax ## encoding: [0x62,0xf4,0x7c,0x08,0xf0,0xc6] +; EGPR-NEXT: retq ## encoding: [0xc3] %tmp = call i64 @llvm.x86.sse42.crc32.64.8(i64 %a, i8 %b) ret i64 %tmp } @@ -20,6 +27,12 @@ define i64 @crc32_64_64(i64 %a, i64 %b) nounwind { ; CHECK-NEXT: movq %rdi, %rax ## encoding: [0x48,0x89,0xf8] ; CHECK-NEXT: crc32q %rsi, %rax ## encoding: [0xf2,0x48,0x0f,0x38,0xf1,0xc6] ; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: crc32_64_64: +; EGPR: ## %bb.0: +; EGPR-NEXT: movq %rdi, %rax ## encoding: [0x48,0x89,0xf8] +; EGPR-NEXT: crc32q %rsi, %rax ## encoding: [0x62,0xf4,0xfc,0x08,0xf1,0xc6] +; EGPR-NEXT: retq ## encoding: [0xc3] %tmp = call i64 @llvm.x86.sse42.crc32.64.64(i64 %a, i64 %b) ret i64 %tmp } diff --git a/llvm/test/CodeGen/X86/invpcid-intrinsic.ll b/llvm/test/CodeGen/X86/invpcid-intrinsic.ll index 3aa9fde35e23..19a6249fc708 100644 --- a/llvm/test/CodeGen/X86/invpcid-intrinsic.ll +++ b/llvm/test/CodeGen/X86/invpcid-intrinsic.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+invpcid | FileCheck %s --check-prefix=X86 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+invpcid | FileCheck %s --check-prefix=X86_64 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+invpcid --show-mc-encoding | FileCheck %s --check-prefix=X86_64 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+invpcid,+egpr --show-mc-encoding | FileCheck %s --check-prefix=EGPR define void @test_invpcid(i32 %type, ptr %descriptor) { ; X86-LABEL: test_invpcid: @@ -12,9 +13,15 @@ define void @test_invpcid(i32 %type, ptr %descriptor) { ; ; X86_64-LABEL: test_invpcid: ; X86_64: # %bb.0: # %entry -; X86_64-NEXT: movl %edi, %eax -; X86_64-NEXT: invpcid (%rsi), %rax -; X86_64-NEXT: retq +; X86_64-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; X86_64-NEXT: invpcid (%rsi), %rax # encoding: [0x66,0x0f,0x38,0x82,0x06] +; X86_64-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_invpcid: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: movl %edi, %eax # encoding: [0x89,0xf8] +; EGPR-NEXT: invpcid (%rsi), %rax # encoding: [0x62,0xf4,0x7e,0x08,0xf2,0x06] +; EGPR-NEXT: retq # encoding: [0xc3] entry: call void @llvm.x86.invpcid(i32 %type, ptr %descriptor) ret void @@ -31,9 +38,15 @@ define void @test_invpcid2(ptr readonly %type, ptr %descriptor) { ; ; X86_64-LABEL: test_invpcid2: ; X86_64: # %bb.0: # %entry -; X86_64-NEXT: movl (%rdi), %eax -; X86_64-NEXT: invpcid (%rsi), %rax -; X86_64-NEXT: retq +; X86_64-NEXT: movl (%rdi), %eax # encoding: [0x8b,0x07] +; X86_64-NEXT: invpcid (%rsi), %rax # encoding: [0x66,0x0f,0x38,0x82,0x06] +; X86_64-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_invpcid2: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: movl (%rdi), %eax # encoding: [0x8b,0x07] +; EGPR-NEXT: invpcid (%rsi), %rax # encoding: [0x62,0xf4,0x7e,0x08,0xf2,0x06] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load i32, ptr %type, align 4 tail call void @llvm.x86.invpcid(i32 %0, ptr %descriptor) #1 diff --git a/llvm/test/CodeGen/X86/movdir-intrinsic-x86.ll b/llvm/test/CodeGen/X86/movdir-intrinsic-x86.ll index 1769bcbf6f60..4d03510ad5d4 100644 --- a/llvm/test/CodeGen/X86/movdir-intrinsic-x86.ll +++ b/llvm/test/CodeGen/X86/movdir-intrinsic-x86.ll @@ -1,19 +1,25 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+movdiri -mattr=+movdir64b | FileCheck %s --check-prefix=X64 ; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+movdiri -mattr=+movdir64b | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+movdiri -mattr=+movdir64b --show-mc-encoding | FileCheck %s --check-prefix=X64 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+movdiri -mattr=+movdir64b,+egpr --show-mc-encoding | FileCheck %s --check-prefix=EGPR define void @test_movdiri(ptr %p, i32 %v) { -; X64-LABEL: test_movdiri: -; X64: # %bb.0: # %entry -; X64-NEXT: movdiri %esi, (%rdi) -; X64-NEXT: retq -; ; X32-LABEL: test_movdiri: ; X32: # %bb.0: # %entry ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X32-NEXT: movdiri %eax, (%ecx) ; X32-NEXT: retl +; +; X64-LABEL: test_movdiri: +; X64: # %bb.0: # %entry +; X64-NEXT: movdiri %esi, (%rdi) # encoding: [0x0f,0x38,0xf9,0x37] +; X64-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_movdiri: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: movdiri %esi, (%rdi) # encoding: [0x62,0xf4,0x7c,0x08,0xf9,0x37] +; EGPR-NEXT: retq # encoding: [0xc3] entry: call void @llvm.x86.directstore32(ptr %p, i32 %v) ret void @@ -22,17 +28,22 @@ entry: declare void @llvm.x86.directstore32(ptr, i32) define void @test_movdir64b(ptr %dst, ptr %src) { -; X64-LABEL: test_movdir64b: -; X64: # %bb.0: # %entry -; X64-NEXT: movdir64b (%rsi), %rdi -; X64-NEXT: retq -; ; X32-LABEL: test_movdir64b: ; X32: # %bb.0: # %entry ; X32-NEXT: movl {{[0-9]+}}(%esp), %eax ; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X32-NEXT: movdir64b (%eax), %ecx ; X32-NEXT: retl +; +; X64-LABEL: test_movdir64b: +; X64: # %bb.0: # %entry +; X64-NEXT: movdir64b (%rsi), %rdi # encoding: [0x66,0x0f,0x38,0xf8,0x3e] +; X64-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_movdir64b: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: movdir64b (%rsi), %rdi # encoding: [0x62,0xf4,0x7d,0x08,0xf8,0x3e] +; EGPR-NEXT: retq # encoding: [0xc3] entry: call void @llvm.x86.movdir64b(ptr %dst, ptr %src) ret void diff --git a/llvm/test/CodeGen/X86/movdir-intrinsic-x86_64.ll b/llvm/test/CodeGen/X86/movdir-intrinsic-x86_64.ll index b20d7df26515..ddd44f6d73d5 100644 --- a/llvm/test/CodeGen/X86/movdir-intrinsic-x86_64.ll +++ b/llvm/test/CodeGen/X86/movdir-intrinsic-x86_64.ll @@ -1,11 +1,17 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+movdiri | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+movdiri --show-mc-encoding | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+movdiri,+egpr --show-mc-encoding | FileCheck %s -check-prefix=EGPR define void @test_movdiri(ptr %p, i64 %v) { ; CHECK-LABEL: test_movdiri: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: movdiri %rsi, (%rdi) -; CHECK-NEXT: retq +; CHECK-NEXT: movdiri %rsi, (%rdi) # encoding: [0x48,0x0f,0x38,0xf9,0x37] +; CHECK-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_movdiri: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: movdiri %rsi, (%rdi) # encoding: [0x62,0xf4,0xfc,0x08,0xf9,0x37] +; EGPR-NEXT: retq # encoding: [0xc3] entry: call void @llvm.x86.directstore64(ptr %p, i64 %v) ret void diff --git a/llvm/test/CodeGen/X86/sha.ll b/llvm/test/CodeGen/X86/sha.ll index d5427556dc0b..d8fa354a3913 100644 --- a/llvm/test/CodeGen/X86/sha.ll +++ b/llvm/test/CodeGen/X86/sha.ll @@ -1,24 +1,45 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mattr=+sha -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK --check-prefix=SSE -; RUN: llc < %s -mattr=+sha,+avx2 -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK --check-prefix=AVX +; RUN: llc < %s -mattr=+sha,+avx2 -mtriple=x86_64-unknown-unknown --show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX +; RUN: llc < %s -mattr=+sha,+egpr -mtriple=x86_64-unknown-unknown --show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=EGPR declare <4 x i32> @llvm.x86.sha1rnds4(<4 x i32>, <4 x i32>, i8) nounwind readnone define <4 x i32> @test_sha1rnds4rr(<4 x i32> %a, <4 x i32> %b) nounwind uwtable { -; CHECK-LABEL: test_sha1rnds4rr: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1rnds4 $3, %xmm1, %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1rnds4rr: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1rnds4 $3, %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1rnds4rr: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1rnds4 $3, %xmm1, %xmm0 # encoding: [0x0f,0x3a,0xcc,0xc1,0x03] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1rnds4rr: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1rnds4 $3, %xmm1, %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xd4,0xc1,0x03] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = tail call <4 x i32> @llvm.x86.sha1rnds4(<4 x i32> %a, <4 x i32> %b, i8 3) ret <4 x i32> %0 } define <4 x i32> @test_sha1rnds4rm(<4 x i32> %a, ptr %b) nounwind uwtable { -; CHECK-LABEL: test_sha1rnds4rm: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1rnds4 $3, (%rdi), %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1rnds4rm: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1rnds4 $3, (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1rnds4rm: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1rnds4 $3, (%rdi), %xmm0 # encoding: [0x0f,0x3a,0xcc,0x07,0x03] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1rnds4rm: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1rnds4 $3, (%rdi), %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xd4,0x07,0x03] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha1rnds4(<4 x i32> %a, <4 x i32> %0, i8 3) @@ -28,20 +49,40 @@ entry: declare <4 x i32> @llvm.x86.sha1nexte(<4 x i32>, <4 x i32>) nounwind readnone define <4 x i32> @test_sha1nexterr(<4 x i32> %a, <4 x i32> %b) nounwind uwtable { -; CHECK-LABEL: test_sha1nexterr: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1nexte %xmm1, %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1nexterr: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1nexte %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1nexterr: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1nexte %xmm1, %xmm0 # encoding: [0x0f,0x38,0xc8,0xc1] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1nexterr: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1nexte %xmm1, %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xd8,0xc1] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = tail call <4 x i32> @llvm.x86.sha1nexte(<4 x i32> %a, <4 x i32> %b) ret <4 x i32> %0 } define <4 x i32> @test_sha1nexterm(<4 x i32> %a, ptr %b) nounwind uwtable { -; CHECK-LABEL: test_sha1nexterm: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1nexte (%rdi), %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1nexterm: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1nexte (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1nexterm: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1nexte (%rdi), %xmm0 # encoding: [0x0f,0x38,0xc8,0x07] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1nexterm: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1nexte (%rdi), %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xd8,0x07] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha1nexte(<4 x i32> %a, <4 x i32> %0) @@ -51,20 +92,40 @@ entry: declare <4 x i32> @llvm.x86.sha1msg1(<4 x i32>, <4 x i32>) nounwind readnone define <4 x i32> @test_sha1msg1rr(<4 x i32> %a, <4 x i32> %b) nounwind uwtable { -; CHECK-LABEL: test_sha1msg1rr: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1msg1 %xmm1, %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1msg1rr: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1msg1 %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1msg1rr: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1msg1 %xmm1, %xmm0 # encoding: [0x0f,0x38,0xc9,0xc1] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1msg1rr: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1msg1 %xmm1, %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xd9,0xc1] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = tail call <4 x i32> @llvm.x86.sha1msg1(<4 x i32> %a, <4 x i32> %b) ret <4 x i32> %0 } define <4 x i32> @test_sha1msg1rm(<4 x i32> %a, ptr %b) nounwind uwtable { -; CHECK-LABEL: test_sha1msg1rm: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1msg1 (%rdi), %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1msg1rm: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1msg1 (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1msg1rm: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1msg1 (%rdi), %xmm0 # encoding: [0x0f,0x38,0xc9,0x07] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1msg1rm: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1msg1 (%rdi), %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xd9,0x07] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha1msg1(<4 x i32> %a, <4 x i32> %0) @@ -74,20 +135,40 @@ entry: declare <4 x i32> @llvm.x86.sha1msg2(<4 x i32>, <4 x i32>) nounwind readnone define <4 x i32> @test_sha1msg2rr(<4 x i32> %a, <4 x i32> %b) nounwind uwtable { -; CHECK-LABEL: test_sha1msg2rr: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1msg2 %xmm1, %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1msg2rr: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1msg2 %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1msg2rr: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1msg2 %xmm1, %xmm0 # encoding: [0x0f,0x38,0xca,0xc1] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1msg2rr: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1msg2 %xmm1, %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xda,0xc1] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = tail call <4 x i32> @llvm.x86.sha1msg2(<4 x i32> %a, <4 x i32> %b) ret <4 x i32> %0 } define <4 x i32> @test_sha1msg2rm(<4 x i32> %a, ptr %b) nounwind uwtable { -; CHECK-LABEL: test_sha1msg2rm: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha1msg2 (%rdi), %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha1msg2rm: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha1msg2 (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha1msg2rm: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha1msg2 (%rdi), %xmm0 # encoding: [0x0f,0x38,0xca,0x07] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1msg2rm: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1msg2 (%rdi), %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xda,0x07] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha1msg2(<4 x i32> %a, <4 x i32> %0) @@ -107,11 +188,19 @@ define <4 x i32> @test_sha256rnds2rr(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) n ; ; AVX-LABEL: test_sha256rnds2rr: ; AVX: # %bb.0: # %entry -; AVX-NEXT: vmovaps %xmm0, %xmm3 -; AVX-NEXT: vmovaps %xmm2, %xmm0 -; AVX-NEXT: sha256rnds2 %xmm0, %xmm1, %xmm3 -; AVX-NEXT: vmovaps %xmm3, %xmm0 -; AVX-NEXT: retq +; AVX-NEXT: vmovaps %xmm0, %xmm3 # encoding: [0xc5,0xf8,0x28,0xd8] +; AVX-NEXT: vmovaps %xmm2, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc2] +; AVX-NEXT: sha256rnds2 %xmm0, %xmm1, %xmm3 # encoding: [0x0f,0x38,0xcb,0xd9] +; AVX-NEXT: vmovaps %xmm3, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc3] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha256rnds2rr: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: movaps %xmm0, %xmm3 # encoding: [0x0f,0x28,0xd8] +; EGPR-NEXT: movaps %xmm2, %xmm0 # encoding: [0x0f,0x28,0xc2] +; EGPR-NEXT: sha256rnds2 %xmm0, %xmm1, %xmm3 # encoding: [0x62,0xf4,0x7c,0x08,0xdb,0xd9] +; EGPR-NEXT: movaps %xmm3, %xmm0 # encoding: [0x0f,0x28,0xc3] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = tail call <4 x i32> @llvm.x86.sha256rnds2(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) ret <4 x i32> %0 @@ -128,11 +217,19 @@ define <4 x i32> @test_sha256rnds2rm(<4 x i32> %a, ptr %b, <4 x i32> %c) nounwin ; ; AVX-LABEL: test_sha256rnds2rm: ; AVX: # %bb.0: # %entry -; AVX-NEXT: vmovaps %xmm0, %xmm2 -; AVX-NEXT: vmovaps %xmm1, %xmm0 -; AVX-NEXT: sha256rnds2 %xmm0, (%rdi), %xmm2 -; AVX-NEXT: vmovaps %xmm2, %xmm0 -; AVX-NEXT: retq +; AVX-NEXT: vmovaps %xmm0, %xmm2 # encoding: [0xc5,0xf8,0x28,0xd0] +; AVX-NEXT: vmovaps %xmm1, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc1] +; AVX-NEXT: sha256rnds2 %xmm0, (%rdi), %xmm2 # encoding: [0x0f,0x38,0xcb,0x17] +; AVX-NEXT: vmovaps %xmm2, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc2] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha256rnds2rm: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: movaps %xmm0, %xmm2 # encoding: [0x0f,0x28,0xd0] +; EGPR-NEXT: movaps %xmm1, %xmm0 # encoding: [0x0f,0x28,0xc1] +; EGPR-NEXT: sha256rnds2 %xmm0, (%rdi), %xmm2 # encoding: [0x62,0xf4,0x7c,0x08,0xdb,0x17] +; EGPR-NEXT: movaps %xmm2, %xmm0 # encoding: [0x0f,0x28,0xc2] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha256rnds2(<4 x i32> %a, <4 x i32> %0, <4 x i32> %c) @@ -142,20 +239,40 @@ entry: declare <4 x i32> @llvm.x86.sha256msg1(<4 x i32>, <4 x i32>) nounwind readnone define <4 x i32> @test_sha256msg1rr(<4 x i32> %a, <4 x i32> %b) nounwind uwtable { -; CHECK-LABEL: test_sha256msg1rr: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha256msg1 %xmm1, %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha256msg1rr: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha256msg1 %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha256msg1rr: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha256msg1 %xmm1, %xmm0 # encoding: [0x0f,0x38,0xcc,0xc1] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha256msg1rr: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha256msg1 %xmm1, %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xdc,0xc1] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = tail call <4 x i32> @llvm.x86.sha256msg1(<4 x i32> %a, <4 x i32> %b) ret <4 x i32> %0 } define <4 x i32> @test_sha256msg1rm(<4 x i32> %a, ptr %b) nounwind uwtable { -; CHECK-LABEL: test_sha256msg1rm: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha256msg1 (%rdi), %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha256msg1rm: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha256msg1 (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha256msg1rm: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha256msg1 (%rdi), %xmm0 # encoding: [0x0f,0x38,0xcc,0x07] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha256msg1rm: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha256msg1 (%rdi), %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xdc,0x07] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha256msg1(<4 x i32> %a, <4 x i32> %0) @@ -165,20 +282,40 @@ entry: declare <4 x i32> @llvm.x86.sha256msg2(<4 x i32>, <4 x i32>) nounwind readnone define <4 x i32> @test_sha256msg2rr(<4 x i32> %a, <4 x i32> %b) nounwind uwtable { -; CHECK-LABEL: test_sha256msg2rr: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha256msg2 %xmm1, %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha256msg2rr: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha256msg2 %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha256msg2rr: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha256msg2 %xmm1, %xmm0 # encoding: [0x0f,0x38,0xcd,0xc1] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha256msg2rr: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha256msg2 %xmm1, %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xdd,0xc1] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = tail call <4 x i32> @llvm.x86.sha256msg2(<4 x i32> %a, <4 x i32> %b) ret <4 x i32> %0 } define <4 x i32> @test_sha256msg2rm(<4 x i32> %a, ptr %b) nounwind uwtable { -; CHECK-LABEL: test_sha256msg2rm: -; CHECK: # %bb.0: # %entry -; CHECK-NEXT: sha256msg2 (%rdi), %xmm0 -; CHECK-NEXT: retq +; SSE-LABEL: test_sha256msg2rm: +; SSE: # %bb.0: # %entry +; SSE-NEXT: sha256msg2 (%rdi), %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: test_sha256msg2rm: +; AVX: # %bb.0: # %entry +; AVX-NEXT: sha256msg2 (%rdi), %xmm0 # encoding: [0x0f,0x38,0xcd,0x07] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha256msg2rm: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha256msg2 (%rdi), %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xdd,0x07] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha256msg2(<4 x i32> %a, <4 x i32> %0) @@ -195,12 +332,20 @@ define <8 x i32> @test_sha1rnds4_zero_extend(<4 x i32> %a, ptr %b) nounwind uwta ; ; AVX-LABEL: test_sha1rnds4_zero_extend: ; AVX: # %bb.0: # %entry -; AVX-NEXT: sha1rnds4 $3, (%rdi), %xmm0 -; AVX-NEXT: vmovaps %xmm0, %xmm0 -; AVX-NEXT: retq +; AVX-NEXT: sha1rnds4 $3, (%rdi), %xmm0 # encoding: [0x0f,0x3a,0xcc,0x07,0x03] +; AVX-NEXT: vmovaps %xmm0, %xmm0 # encoding: [0xc5,0xf8,0x28,0xc0] +; AVX-NEXT: retq # encoding: [0xc3] +; +; EGPR-LABEL: test_sha1rnds4_zero_extend: +; EGPR: # %bb.0: # %entry +; EGPR-NEXT: sha1rnds4 $3, (%rdi), %xmm0 # encoding: [0x62,0xf4,0x7c,0x08,0xd4,0x07,0x03] +; EGPR-NEXT: xorps %xmm1, %xmm1 # encoding: [0x0f,0x57,0xc9] +; EGPR-NEXT: retq # encoding: [0xc3] entry: %0 = load <4 x i32>, ptr %b %1 = tail call <4 x i32> @llvm.x86.sha1rnds4(<4 x i32> %a, <4 x i32> %0, i8 3) %2 = shufflevector <4 x i32> %1, <4 x i32> zeroinitializer, <8 x i32> ret <8 x i32> %2 } +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}} diff --git a/llvm/test/CodeGen/X86/x64-cet-intrinsics.ll b/llvm/test/CodeGen/X86/x64-cet-intrinsics.ll index 4c28c8ab4369..bf87ae5cac05 100644 --- a/llvm/test/CodeGen/X86/x64-cet-intrinsics.ll +++ b/llvm/test/CodeGen/X86/x64-cet-intrinsics.ll @@ -1,11 +1,17 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+shstk | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+shstk --show-mc-encoding | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+shstk,+egpr --show-mc-encoding | FileCheck %s -check-prefix=EGPR define void @test_incsspd(i32 %a) local_unnamed_addr { ; CHECK-LABEL: test_incsspd: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: incsspd %edi -; CHECK-NEXT: retq +; CHECK-NEXT: incsspd %edi ## encoding: [0xf3,0x0f,0xae,0xef] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_incsspd: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: incsspd %edi ## encoding: [0xf3,0x0f,0xae,0xef] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.incsspd(i32 %a) ret void @@ -16,9 +22,15 @@ declare void @llvm.x86.incsspd(i32) define void @test_incsspq(i32 %a) local_unnamed_addr { ; CHECK-LABEL: test_incsspq: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: movslq %edi, %rax -; CHECK-NEXT: incsspq %rax -; CHECK-NEXT: retq +; CHECK-NEXT: movslq %edi, %rax ## encoding: [0x48,0x63,0xc7] +; CHECK-NEXT: incsspq %rax ## encoding: [0xf3,0x48,0x0f,0xae,0xe8] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_incsspq: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: movslq %edi, %rax ## encoding: [0x48,0x63,0xc7] +; EGPR-NEXT: incsspq %rax ## encoding: [0xf3,0x48,0x0f,0xae,0xe8] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: %conv.i = sext i32 %a to i64 tail call void @llvm.x86.incsspq(i64 %conv.i) @@ -30,9 +42,15 @@ declare void @llvm.x86.incsspq(i64) define i32 @test_rdsspd(i32 %a) { ; CHECK-LABEL: test_rdsspd: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: movl %edi, %eax -; CHECK-NEXT: rdsspd %eax -; CHECK-NEXT: retq +; CHECK-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] +; CHECK-NEXT: rdsspd %eax ## encoding: [0xf3,0x0f,0x1e,0xc8] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_rdsspd: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: movl %edi, %eax ## encoding: [0x89,0xf8] +; EGPR-NEXT: rdsspd %eax ## encoding: [0xf3,0x0f,0x1e,0xc8] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: %0 = call i32 @llvm.x86.rdsspd(i32 %a) ret i32 %0 @@ -43,9 +61,15 @@ declare i32 @llvm.x86.rdsspd(i32) define i64 @test_rdsspq(i64 %a) { ; CHECK-LABEL: test_rdsspq: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: movq %rdi, %rax -; CHECK-NEXT: rdsspq %rax -; CHECK-NEXT: retq +; CHECK-NEXT: movq %rdi, %rax ## encoding: [0x48,0x89,0xf8] +; CHECK-NEXT: rdsspq %rax ## encoding: [0xf3,0x48,0x0f,0x1e,0xc8] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_rdsspq: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: movq %rdi, %rax ## encoding: [0x48,0x89,0xf8] +; EGPR-NEXT: rdsspq %rax ## encoding: [0xf3,0x48,0x0f,0x1e,0xc8] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: %0 = call i64 @llvm.x86.rdsspq(i64 %a) ret i64 %0 @@ -56,8 +80,13 @@ declare i64 @llvm.x86.rdsspq(i64) define void @test_saveprevssp() { ; CHECK-LABEL: test_saveprevssp: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: saveprevssp -; CHECK-NEXT: retq +; CHECK-NEXT: saveprevssp ## encoding: [0xf3,0x0f,0x01,0xea] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_saveprevssp: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: saveprevssp ## encoding: [0xf3,0x0f,0x01,0xea] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.saveprevssp() ret void @@ -68,8 +97,13 @@ declare void @llvm.x86.saveprevssp() define void @test_rstorssp(ptr %__p) { ; CHECK-LABEL: test_rstorssp: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: rstorssp (%rdi) -; CHECK-NEXT: retq +; CHECK-NEXT: rstorssp (%rdi) ## encoding: [0xf3,0x0f,0x01,0x2f] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_rstorssp: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: rstorssp (%rdi) ## encoding: [0xf3,0x0f,0x01,0x2f] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.rstorssp(ptr %__p) ret void @@ -80,8 +114,13 @@ declare void @llvm.x86.rstorssp(ptr) define void @test_wrssd(i32 %a, ptr %__p) { ; CHECK-LABEL: test_wrssd: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: wrssd %edi, (%rsi) -; CHECK-NEXT: retq +; CHECK-NEXT: wrssd %edi, (%rsi) ## encoding: [0x0f,0x38,0xf6,0x3e] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_wrssd: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: wrssd %edi, (%rsi) ## encoding: [0x62,0xf4,0x7c,0x08,0x66,0x3e] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.wrssd(i32 %a, ptr %__p) ret void @@ -92,8 +131,13 @@ declare void @llvm.x86.wrssd(i32, ptr) define void @test_wrssq(i64 %a, ptr %__p) { ; CHECK-LABEL: test_wrssq: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: wrssq %rdi, (%rsi) -; CHECK-NEXT: retq +; CHECK-NEXT: wrssq %rdi, (%rsi) ## encoding: [0x48,0x0f,0x38,0xf6,0x3e] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_wrssq: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: wrssq %rdi, (%rsi) ## encoding: [0x62,0xf4,0xfc,0x08,0x66,0x3e] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.wrssq(i64 %a, ptr %__p) ret void @@ -104,8 +148,13 @@ declare void @llvm.x86.wrssq(i64, ptr) define void @test_wrussd(i32 %a, ptr %__p) { ; CHECK-LABEL: test_wrussd: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: wrussd %edi, (%rsi) -; CHECK-NEXT: retq +; CHECK-NEXT: wrussd %edi, (%rsi) ## encoding: [0x66,0x0f,0x38,0xf5,0x3e] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_wrussd: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: wrussd %edi, (%rsi) ## encoding: [0x62,0xf4,0x7d,0x08,0x65,0x3e] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.wrussd(i32 %a, ptr %__p) ret void @@ -116,8 +165,13 @@ declare void @llvm.x86.wrussd(i32, ptr) define void @test_wrussq(i64 %a, ptr %__p) { ; CHECK-LABEL: test_wrussq: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: wrussq %rdi, (%rsi) -; CHECK-NEXT: retq +; CHECK-NEXT: wrussq %rdi, (%rsi) ## encoding: [0x66,0x48,0x0f,0x38,0xf5,0x3e] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_wrussq: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: wrussq %rdi, (%rsi) ## encoding: [0x62,0xf4,0xfd,0x08,0x65,0x3e] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.wrussq(i64 %a, ptr %__p) ret void @@ -128,8 +182,13 @@ declare void @llvm.x86.wrussq(i64, ptr) define void @test_setssbsy() { ; CHECK-LABEL: test_setssbsy: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: setssbsy -; CHECK-NEXT: retq +; CHECK-NEXT: setssbsy ## encoding: [0xf3,0x0f,0x01,0xe8] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_setssbsy: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: setssbsy ## encoding: [0xf3,0x0f,0x01,0xe8] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.setssbsy() ret void @@ -140,8 +199,13 @@ declare void @llvm.x86.setssbsy() define void @test_clrssbsy(ptr %__p) { ; CHECK-LABEL: test_clrssbsy: ; CHECK: ## %bb.0: ## %entry -; CHECK-NEXT: clrssbsy (%rdi) -; CHECK-NEXT: retq +; CHECK-NEXT: clrssbsy (%rdi) ## encoding: [0xf3,0x0f,0xae,0x37] +; CHECK-NEXT: retq ## encoding: [0xc3] +; +; EGPR-LABEL: test_clrssbsy: +; EGPR: ## %bb.0: ## %entry +; EGPR-NEXT: clrssbsy (%rdi) ## encoding: [0xf3,0x0f,0xae,0x37] +; EGPR-NEXT: retq ## encoding: [0xc3] entry: tail call void @llvm.x86.clrssbsy(ptr %__p) ret void -- GitLab From 77b124cc574bb427b066fa8c56df7d37a27feb8f Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 5 Jan 2024 08:11:44 +0000 Subject: [PATCH 074/728] [AArch64][GlobalISel] Add legalization for G_VECREDUCE_SEQ_FADD. (#76238) And G_VECREDUCE_SEQ_FMUL at the same time. They require the elements of the vector operand to be accumulated in order, so just need to be scalarized. Some of the operands are not simplified as much as they can quite yet due to not canonicalizing constant operands post-legalization. --- .../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 3 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 33 ++ .../AArch64/GISel/AArch64LegalizerInfo.cpp | 4 + .../GlobalISel/legalizer-info-validation.mir | 9 +- .../CodeGen/AArch64/vecreduce-fadd-strict.ll | 406 +++++++++++++++--- .../CodeGen/AArch64/vecreduce-fmul-strict.ll | 315 ++++++++++++-- 6 files changed, 681 insertions(+), 89 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 916e6dda3991..586679fa2954 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -350,6 +350,9 @@ public: LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + LegalizeResult fewerElementsVectorSeqReductions(MachineInstr &MI, + unsigned TypeIdx, + LLT NarrowTy); LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 3de8fee1ac4e..def7f6ebeb01 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -4718,6 +4718,9 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, return fewerElementsVectorMultiEltType(GMI, NumElts, {2 /*imm*/}); GISEL_VECREDUCE_CASES_NONSEQ return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy); + case TargetOpcode::G_VECREDUCE_SEQ_FADD: + case TargetOpcode::G_VECREDUCE_SEQ_FMUL: + return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy); case G_SHUFFLE_VECTOR: return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy); case G_FPOWI: @@ -4951,6 +4954,36 @@ LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions( return Legalized; } +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI, + unsigned int TypeIdx, + LLT NarrowTy) { + auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] = + MI.getFirst3RegLLTs(); + if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy || + DstTy != NarrowTy) + return UnableToLegalize; + + assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD || + MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) && + "Unexpected vecreduce opcode"); + unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD + ? TargetOpcode::G_FADD + : TargetOpcode::G_FMUL; + + SmallVector SplitSrcs; + unsigned NumParts = SrcTy.getNumElements(); + extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs); + Register Acc = ScalarReg; + for (unsigned i = 0; i < NumParts; i++) + Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[i]}) + .getReg(0); + + MIRBuilder.buildCopy(DstReg, Acc); + MI.eraseFromParent(); + return Legalized; +} + LegalizerHelper::LegalizeResult LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, LLT SrcTy, LLT NarrowTy, diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 4ba73736294f..470742cdc30e 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -1024,6 +1024,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .scalarize(1) .lower(); + getActionDefinitionsBuilder({G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL}) + .scalarize(2) + .lower(); + getActionDefinitionsBuilder(G_VECREDUCE_ADD) .legalFor({{s8, v16s8}, {s8, v8s8}, diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index df1c5ff04043..2adf9763a96b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -726,11 +726,12 @@ # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices -# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined -# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: G_VECREDUCE_SEQ_FMUL (opcode {{[0-9]+}}): 3 type indices, 0 imm indices -# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined -# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}} +# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: G_VECREDUCE_FADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll index 63b5a97703e6..20237705cc6e 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd-strict.ll @@ -1,13 +1,24 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 ; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define float @add_HalfS(<2 x float> %bin.rdx) { -; CHECK-LABEL: add_HalfS: -; CHECK: // %bb.0: -; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-NEXT: faddp s0, v0.2s -; CHECK-NEXT: ret +; CHECK-SD-LABEL: add_HalfS: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-SD-NEXT: faddp s0, v0.2s +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_HalfS: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2s, #128, lsl #24 +; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-NEXT: mov s2, v0.s[1] +; CHECK-GI-NEXT: fadd s0, s1, s0 +; CHECK-GI-NEXT: fadd s0, s0, s2 +; CHECK-GI-NEXT: ret %r = call float @llvm.vector.reduce.fadd.f32.v2f32(float -0.0, <2 x float> %bin.rdx) ret float %r } @@ -42,6 +53,46 @@ define half @add_HalfH(<4 x half> %bin.rdx) { ; CHECK-SD-FP16-NEXT: fadd h1, h2, h1 ; CHECK-SD-FP16-NEXT: fadd h0, h1, h0 ; CHECK-SD-FP16-NEXT: ret +; +; CHECK-GI-NOFP16-LABEL: add_HalfH: +; CHECK-GI-NOFP16: // %bb.0: +; CHECK-GI-NOFP16-NEXT: mov w8, #32768 // =0x8000 +; CHECK-GI-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h0 +; CHECK-GI-NOFP16-NEXT: fmov s1, w8 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[2] +; CHECK-GI-NOFP16-NEXT: mov h0, v0.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s0, s1, s0 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: ret +; +; CHECK-GI-FP16-LABEL: add_HalfH: +; CHECK-GI-FP16: // %bb.0: +; CHECK-GI-FP16-NEXT: adrp x8, .LCPI1_0 +; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-FP16-NEXT: mov h2, v0.h[1] +; CHECK-GI-FP16-NEXT: mov h3, v0.h[2] +; CHECK-GI-FP16-NEXT: ldr h1, [x8, :lo12:.LCPI1_0] +; CHECK-GI-FP16-NEXT: fadd h1, h1, h0 +; CHECK-GI-FP16-NEXT: mov h0, v0.h[3] +; CHECK-GI-FP16-NEXT: fadd h1, h1, h2 +; CHECK-GI-FP16-NEXT: fadd h1, h1, h3 +; CHECK-GI-FP16-NEXT: fadd h0, h1, h0 +; CHECK-GI-FP16-NEXT: ret %r = call half @llvm.vector.reduce.fadd.f16.v4f16(half -0.0, <4 x half> %bin.rdx) ret half %r } @@ -103,28 +154,115 @@ define half @add_H(<8 x half> %bin.rdx) { ; CHECK-SD-FP16-NEXT: fadd h1, h1, h2 ; CHECK-SD-FP16-NEXT: fadd h0, h1, h0 ; CHECK-SD-FP16-NEXT: ret +; +; CHECK-GI-NOFP16-LABEL: add_H: +; CHECK-GI-NOFP16: // %bb.0: +; CHECK-GI-NOFP16-NEXT: mov w8, #32768 // =0x8000 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h0 +; CHECK-GI-NOFP16-NEXT: fmov s1, w8 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[2] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[4] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[5] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[6] +; CHECK-GI-NOFP16-NEXT: mov h0, v0.h[7] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fadd s0, s1, s0 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: ret +; +; CHECK-GI-FP16-LABEL: add_H: +; CHECK-GI-FP16: // %bb.0: +; CHECK-GI-FP16-NEXT: adrp x8, .LCPI2_0 +; CHECK-GI-FP16-NEXT: mov h2, v0.h[1] +; CHECK-GI-FP16-NEXT: mov h3, v0.h[2] +; CHECK-GI-FP16-NEXT: ldr h1, [x8, :lo12:.LCPI2_0] +; CHECK-GI-FP16-NEXT: fadd h1, h1, h0 +; CHECK-GI-FP16-NEXT: fadd h1, h1, h2 +; CHECK-GI-FP16-NEXT: mov h2, v0.h[3] +; CHECK-GI-FP16-NEXT: fadd h1, h1, h3 +; CHECK-GI-FP16-NEXT: mov h3, v0.h[4] +; CHECK-GI-FP16-NEXT: fadd h1, h1, h2 +; CHECK-GI-FP16-NEXT: mov h2, v0.h[5] +; CHECK-GI-FP16-NEXT: fadd h1, h1, h3 +; CHECK-GI-FP16-NEXT: mov h3, v0.h[6] +; CHECK-GI-FP16-NEXT: mov h0, v0.h[7] +; CHECK-GI-FP16-NEXT: fadd h1, h1, h2 +; CHECK-GI-FP16-NEXT: fadd h1, h1, h3 +; CHECK-GI-FP16-NEXT: fadd h0, h1, h0 +; CHECK-GI-FP16-NEXT: ret %r = call half @llvm.vector.reduce.fadd.f16.v8f16(half -0.0, <8 x half> %bin.rdx) ret half %r } define float @add_S(<4 x float> %bin.rdx) { -; CHECK-LABEL: add_S: -; CHECK: // %bb.0: -; CHECK-NEXT: mov s1, v0.s[2] -; CHECK-NEXT: faddp s2, v0.2s -; CHECK-NEXT: mov s0, v0.s[3] -; CHECK-NEXT: fadd s1, s2, s1 -; CHECK-NEXT: fadd s0, s1, s0 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: add_S: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: mov s1, v0.s[2] +; CHECK-SD-NEXT: faddp s2, v0.2s +; CHECK-SD-NEXT: mov s0, v0.s[3] +; CHECK-SD-NEXT: fadd s1, s2, s1 +; CHECK-SD-NEXT: fadd s0, s1, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_S: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2s, #128, lsl #24 +; CHECK-GI-NEXT: mov s2, v0.s[1] +; CHECK-GI-NEXT: mov s3, v0.s[2] +; CHECK-GI-NEXT: fadd s1, s1, s0 +; CHECK-GI-NEXT: mov s0, v0.s[3] +; CHECK-GI-NEXT: fadd s1, s1, s2 +; CHECK-GI-NEXT: fadd s1, s1, s3 +; CHECK-GI-NEXT: fadd s0, s1, s0 +; CHECK-GI-NEXT: ret %r = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %bin.rdx) ret float %r } define double @add_D(<2 x double> %bin.rdx) { -; CHECK-LABEL: add_D: -; CHECK: // %bb.0: -; CHECK-NEXT: faddp d0, v0.2d -; CHECK-NEXT: ret +; CHECK-SD-LABEL: add_D: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: faddp d0, v0.2d +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_D: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000 +; CHECK-GI-NEXT: mov d2, v0.d[1] +; CHECK-GI-NEXT: fmov d1, x8 +; CHECK-GI-NEXT: fadd d0, d1, d0 +; CHECK-GI-NEXT: fadd d0, d0, d2 +; CHECK-GI-NEXT: ret %r = call double @llvm.vector.reduce.fadd.f64.v2f64(double -0.0, <2 x double> %bin.rdx) ret double %r } @@ -239,56 +377,220 @@ define half @add_2H(<16 x half> %bin.rdx) { ; CHECK-SD-FP16-NEXT: fadd h0, h0, h3 ; CHECK-SD-FP16-NEXT: fadd h0, h0, h1 ; CHECK-SD-FP16-NEXT: ret +; +; CHECK-GI-NOFP16-LABEL: add_2H: +; CHECK-GI-NOFP16: // %bb.0: +; CHECK-GI-NOFP16-NEXT: mov w8, #32768 // =0x8000 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h0 +; CHECK-GI-NOFP16-NEXT: fmov s2, w8 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[2] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[4] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[5] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[6] +; CHECK-GI-NOFP16-NEXT: mov h0, v0.h[7] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fadd s0, s2, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h1 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[2] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[4] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[5] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[6] +; CHECK-GI-NOFP16-NEXT: mov h1, v1.h[7] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s1 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: ret +; +; CHECK-GI-FP16-LABEL: add_2H: +; CHECK-GI-FP16: // %bb.0: +; CHECK-GI-FP16-NEXT: adrp x8, .LCPI5_0 +; CHECK-GI-FP16-NEXT: mov h3, v0.h[1] +; CHECK-GI-FP16-NEXT: mov h4, v0.h[2] +; CHECK-GI-FP16-NEXT: ldr h2, [x8, :lo12:.LCPI5_0] +; CHECK-GI-FP16-NEXT: fadd h2, h2, h0 +; CHECK-GI-FP16-NEXT: fadd h2, h2, h3 +; CHECK-GI-FP16-NEXT: mov h3, v0.h[3] +; CHECK-GI-FP16-NEXT: fadd h2, h2, h4 +; CHECK-GI-FP16-NEXT: mov h4, v0.h[4] +; CHECK-GI-FP16-NEXT: fadd h2, h2, h3 +; CHECK-GI-FP16-NEXT: mov h3, v0.h[5] +; CHECK-GI-FP16-NEXT: fadd h2, h2, h4 +; CHECK-GI-FP16-NEXT: mov h4, v0.h[6] +; CHECK-GI-FP16-NEXT: mov h0, v0.h[7] +; CHECK-GI-FP16-NEXT: fadd h2, h2, h3 +; CHECK-GI-FP16-NEXT: mov h3, v1.h[2] +; CHECK-GI-FP16-NEXT: fadd h2, h2, h4 +; CHECK-GI-FP16-NEXT: fadd h0, h2, h0 +; CHECK-GI-FP16-NEXT: mov h2, v1.h[1] +; CHECK-GI-FP16-NEXT: fadd h0, h0, h1 +; CHECK-GI-FP16-NEXT: fadd h0, h0, h2 +; CHECK-GI-FP16-NEXT: mov h2, v1.h[3] +; CHECK-GI-FP16-NEXT: fadd h0, h0, h3 +; CHECK-GI-FP16-NEXT: mov h3, v1.h[4] +; CHECK-GI-FP16-NEXT: fadd h0, h0, h2 +; CHECK-GI-FP16-NEXT: mov h2, v1.h[5] +; CHECK-GI-FP16-NEXT: fadd h0, h0, h3 +; CHECK-GI-FP16-NEXT: mov h3, v1.h[6] +; CHECK-GI-FP16-NEXT: mov h1, v1.h[7] +; CHECK-GI-FP16-NEXT: fadd h0, h0, h2 +; CHECK-GI-FP16-NEXT: fadd h0, h0, h3 +; CHECK-GI-FP16-NEXT: fadd h0, h0, h1 +; CHECK-GI-FP16-NEXT: ret %r = call half @llvm.vector.reduce.fadd.f16.v16f16(half -0.0, <16 x half> %bin.rdx) ret half %r } define float @add_2S(<8 x float> %bin.rdx) { -; CHECK-LABEL: add_2S: -; CHECK: // %bb.0: -; CHECK-NEXT: mov s2, v0.s[2] -; CHECK-NEXT: faddp s3, v0.2s -; CHECK-NEXT: mov s0, v0.s[3] -; CHECK-NEXT: fadd s2, s3, s2 -; CHECK-NEXT: mov s3, v1.s[2] -; CHECK-NEXT: fadd s0, s2, s0 -; CHECK-NEXT: mov s2, v1.s[1] -; CHECK-NEXT: fadd s0, s0, s1 -; CHECK-NEXT: mov s1, v1.s[3] -; CHECK-NEXT: fadd s0, s0, s2 -; CHECK-NEXT: fadd s0, s0, s3 -; CHECK-NEXT: fadd s0, s0, s1 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: add_2S: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: mov s2, v0.s[2] +; CHECK-SD-NEXT: faddp s3, v0.2s +; CHECK-SD-NEXT: mov s0, v0.s[3] +; CHECK-SD-NEXT: fadd s2, s3, s2 +; CHECK-SD-NEXT: mov s3, v1.s[2] +; CHECK-SD-NEXT: fadd s0, s2, s0 +; CHECK-SD-NEXT: mov s2, v1.s[1] +; CHECK-SD-NEXT: fadd s0, s0, s1 +; CHECK-SD-NEXT: mov s1, v1.s[3] +; CHECK-SD-NEXT: fadd s0, s0, s2 +; CHECK-SD-NEXT: fadd s0, s0, s3 +; CHECK-SD-NEXT: fadd s0, s0, s1 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_2S: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v2.2s, #128, lsl #24 +; CHECK-GI-NEXT: mov s3, v0.s[1] +; CHECK-GI-NEXT: mov s4, v0.s[2] +; CHECK-GI-NEXT: fadd s2, s2, s0 +; CHECK-GI-NEXT: mov s0, v0.s[3] +; CHECK-GI-NEXT: fadd s2, s2, s3 +; CHECK-GI-NEXT: mov s3, v1.s[2] +; CHECK-GI-NEXT: fadd s2, s2, s4 +; CHECK-GI-NEXT: fadd s0, s2, s0 +; CHECK-GI-NEXT: mov s2, v1.s[1] +; CHECK-GI-NEXT: fadd s0, s0, s1 +; CHECK-GI-NEXT: mov s1, v1.s[3] +; CHECK-GI-NEXT: fadd s0, s0, s2 +; CHECK-GI-NEXT: fadd s0, s0, s3 +; CHECK-GI-NEXT: fadd s0, s0, s1 +; CHECK-GI-NEXT: ret %r = call float @llvm.vector.reduce.fadd.f32.v8f32(float -0.0, <8 x float> %bin.rdx) ret float %r } define double @add_2D(<4 x double> %bin.rdx) { -; CHECK-LABEL: add_2D: -; CHECK: // %bb.0: -; CHECK-NEXT: faddp d0, v0.2d -; CHECK-NEXT: mov d2, v1.d[1] -; CHECK-NEXT: fadd d0, d0, d1 -; CHECK-NEXT: fadd d0, d0, d2 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: add_2D: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: faddp d0, v0.2d +; CHECK-SD-NEXT: mov d2, v1.d[1] +; CHECK-SD-NEXT: fadd d0, d0, d1 +; CHECK-SD-NEXT: fadd d0, d0, d2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_2D: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000 +; CHECK-GI-NEXT: mov d3, v0.d[1] +; CHECK-GI-NEXT: fmov d2, x8 +; CHECK-GI-NEXT: fadd d0, d2, d0 +; CHECK-GI-NEXT: mov d2, v1.d[1] +; CHECK-GI-NEXT: fadd d0, d0, d3 +; CHECK-GI-NEXT: fadd d0, d0, d1 +; CHECK-GI-NEXT: fadd d0, d0, d2 +; CHECK-GI-NEXT: ret %r = call double @llvm.vector.reduce.fadd.f64.v4f64(double -0.0, <4 x double> %bin.rdx) ret double %r } ; Added at least one test where the start value is not -0.0. define float @add_S_init_42(<4 x float> %bin.rdx) { -; CHECK-LABEL: add_S_init_42: -; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #1109917696 // =0x42280000 -; CHECK-NEXT: mov s2, v0.s[1] -; CHECK-NEXT: mov s3, v0.s[2] -; CHECK-NEXT: fmov s1, w8 -; CHECK-NEXT: fadd s1, s0, s1 -; CHECK-NEXT: mov s0, v0.s[3] -; CHECK-NEXT: fadd s1, s1, s2 -; CHECK-NEXT: fadd s1, s1, s3 -; CHECK-NEXT: fadd s0, s1, s0 -; CHECK-NEXT: ret +; CHECK-SD-LABEL: add_S_init_42: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: mov w8, #1109917696 // =0x42280000 +; CHECK-SD-NEXT: mov s2, v0.s[1] +; CHECK-SD-NEXT: mov s3, v0.s[2] +; CHECK-SD-NEXT: fmov s1, w8 +; CHECK-SD-NEXT: fadd s1, s0, s1 +; CHECK-SD-NEXT: mov s0, v0.s[3] +; CHECK-SD-NEXT: fadd s1, s1, s2 +; CHECK-SD-NEXT: fadd s1, s1, s3 +; CHECK-SD-NEXT: fadd s0, s1, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: add_S_init_42: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: mov w8, #1109917696 // =0x42280000 +; CHECK-GI-NEXT: mov s2, v0.s[1] +; CHECK-GI-NEXT: mov s3, v0.s[2] +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: fadd s1, s1, s0 +; CHECK-GI-NEXT: mov s0, v0.s[3] +; CHECK-GI-NEXT: fadd s1, s1, s2 +; CHECK-GI-NEXT: fadd s1, s1, s3 +; CHECK-GI-NEXT: fadd s0, s1, s0 +; CHECK-GI-NEXT: ret %r = call float @llvm.vector.reduce.fadd.f32.v4f32(float 42.0, <4 x float> %bin.rdx) ret float %r } @@ -303,4 +605,4 @@ declare float @llvm.vector.reduce.fadd.f32.v8f32(float, <8 x float>) declare double @llvm.vector.reduce.fadd.f64.v2f64(double, <2 x double>) declare double @llvm.vector.reduce.fadd.f64.v4f64(double, <4 x double>) ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; CHECK-SD: {{.*}} +; CHECK: {{.*}} diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll index 68cd3496a923..32ce4d6eb967 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul-strict.ll @@ -1,13 +1,24 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16 ; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16 +; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16 +; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16 define float @mul_HalfS(<2 x float> %bin.rdx) { -; CHECK-LABEL: mul_HalfS: -; CHECK: // %bb.0: -; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-NEXT: fmul s0, s0, v0.s[1] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: mul_HalfS: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-SD-NEXT: fmul s0, s0, v0.s[1] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: mul_HalfS: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov s1, #1.00000000 +; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-NEXT: mov s2, v0.s[1] +; CHECK-GI-NEXT: fmul s0, s1, s0 +; CHECK-GI-NEXT: fmul s0, s0, s2 +; CHECK-GI-NEXT: ret %r = call float @llvm.vector.reduce.fmul.f32.v2f32(float 1.0, <2 x float> %bin.rdx) ret float %r } @@ -40,6 +51,45 @@ define half @mul_HalfH(<4 x half> %bin.rdx) { ; CHECK-SD-FP16-NEXT: fmul h1, h1, v0.h[2] ; CHECK-SD-FP16-NEXT: fmul h0, h1, v0.h[3] ; CHECK-SD-FP16-NEXT: ret +; +; CHECK-GI-NOFP16-LABEL: mul_HalfH: +; CHECK-GI-NOFP16: // %bb.0: +; CHECK-GI-NOFP16-NEXT: mov w8, #15360 // =0x3c00 +; CHECK-GI-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h0 +; CHECK-GI-NOFP16-NEXT: fmov s1, w8 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[2] +; CHECK-GI-NOFP16-NEXT: mov h0, v0.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s0, s1, s0 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: ret +; +; CHECK-GI-FP16-LABEL: mul_HalfH: +; CHECK-GI-FP16: // %bb.0: +; CHECK-GI-FP16-NEXT: fmov h1, #1.00000000 +; CHECK-GI-FP16-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-GI-FP16-NEXT: mov h2, v0.h[1] +; CHECK-GI-FP16-NEXT: mov h3, v0.h[2] +; CHECK-GI-FP16-NEXT: fmul h1, h1, h0 +; CHECK-GI-FP16-NEXT: mov h0, v0.h[3] +; CHECK-GI-FP16-NEXT: fmul h1, h1, h2 +; CHECK-GI-FP16-NEXT: fmul h1, h1, h3 +; CHECK-GI-FP16-NEXT: fmul h0, h1, h0 +; CHECK-GI-FP16-NEXT: ret %r = call half @llvm.vector.reduce.fmul.f16.v4f16(half 1.0, <4 x half> %bin.rdx) ret half %r } @@ -95,26 +145,100 @@ define half @mul_H(<8 x half> %bin.rdx) { ; CHECK-SD-FP16-NEXT: fmul h1, h1, v0.h[6] ; CHECK-SD-FP16-NEXT: fmul h0, h1, v0.h[7] ; CHECK-SD-FP16-NEXT: ret +; +; CHECK-GI-NOFP16-LABEL: mul_H: +; CHECK-GI-NOFP16: // %bb.0: +; CHECK-GI-NOFP16-NEXT: mov w8, #15360 // =0x3c00 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h0 +; CHECK-GI-NOFP16-NEXT: fmov s1, w8 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[2] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[4] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[5] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v0.h[6] +; CHECK-GI-NOFP16-NEXT: mov h0, v0.h[7] +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s1, s1, s2 +; CHECK-GI-NOFP16-NEXT: fcvt h1, s1 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fmul s0, s1, s0 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: ret +; +; CHECK-GI-FP16-LABEL: mul_H: +; CHECK-GI-FP16: // %bb.0: +; CHECK-GI-FP16-NEXT: fmov h1, #1.00000000 +; CHECK-GI-FP16-NEXT: fmul h1, h1, h0 +; CHECK-GI-FP16-NEXT: fmul h1, h1, v0.h[1] +; CHECK-GI-FP16-NEXT: fmul h1, h1, v0.h[2] +; CHECK-GI-FP16-NEXT: fmul h1, h1, v0.h[3] +; CHECK-GI-FP16-NEXT: fmul h1, h1, v0.h[4] +; CHECK-GI-FP16-NEXT: fmul h1, h1, v0.h[5] +; CHECK-GI-FP16-NEXT: fmul h1, h1, v0.h[6] +; CHECK-GI-FP16-NEXT: fmul h0, h1, v0.h[7] +; CHECK-GI-FP16-NEXT: ret %r = call half @llvm.vector.reduce.fmul.f16.v8f16(half 1.0, <8 x half> %bin.rdx) ret half %r } define float @mul_S(<4 x float> %bin.rdx) { -; CHECK-LABEL: mul_S: -; CHECK: // %bb.0: -; CHECK-NEXT: fmul s1, s0, v0.s[1] -; CHECK-NEXT: fmul s1, s1, v0.s[2] -; CHECK-NEXT: fmul s0, s1, v0.s[3] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: mul_S: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: fmul s1, s0, v0.s[1] +; CHECK-SD-NEXT: fmul s1, s1, v0.s[2] +; CHECK-SD-NEXT: fmul s0, s1, v0.s[3] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: mul_S: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov s1, #1.00000000 +; CHECK-GI-NEXT: fmul s1, s1, s0 +; CHECK-GI-NEXT: fmul s1, s1, v0.s[1] +; CHECK-GI-NEXT: fmul s1, s1, v0.s[2] +; CHECK-GI-NEXT: fmul s0, s1, v0.s[3] +; CHECK-GI-NEXT: ret %r = call float @llvm.vector.reduce.fmul.f32.v4f32(float 1.0, <4 x float> %bin.rdx) ret float %r } define double @mul_D(<2 x double> %bin.rdx) { -; CHECK-LABEL: mul_D: -; CHECK: // %bb.0: -; CHECK-NEXT: fmul d0, d0, v0.d[1] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: mul_D: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: fmul d0, d0, v0.d[1] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: mul_D: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov d1, #1.00000000 +; CHECK-GI-NEXT: fmul d1, d1, d0 +; CHECK-GI-NEXT: fmul d0, d1, v0.d[1] +; CHECK-GI-NEXT: ret %r = call double @llvm.vector.reduce.fmul.f64.v2f64(double 1.0, <2 x double> %bin.rdx) ret double %r } @@ -216,32 +340,159 @@ define half @mul_2H(<16 x half> %bin.rdx) { ; CHECK-SD-FP16-NEXT: fmul h0, h0, v1.h[6] ; CHECK-SD-FP16-NEXT: fmul h0, h0, v1.h[7] ; CHECK-SD-FP16-NEXT: ret +; +; CHECK-GI-NOFP16-LABEL: mul_2H: +; CHECK-GI-NOFP16: // %bb.0: +; CHECK-GI-NOFP16-NEXT: mov w8, #15360 // =0x3c00 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h0 +; CHECK-GI-NOFP16-NEXT: fmov s2, w8 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[2] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[4] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[5] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: mov h3, v0.h[6] +; CHECK-GI-NOFP16-NEXT: mov h0, v0.h[7] +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s3, h3 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s2, s2, s3 +; CHECK-GI-NOFP16-NEXT: fcvt h2, s2 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fmul s0, s2, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h1 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[1] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[2] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[3] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[4] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[5] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: mov h2, v1.h[6] +; CHECK-GI-NOFP16-NEXT: mov h1, v1.h[7] +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s2, h2 +; CHECK-GI-NOFP16-NEXT: fcvt s1, h1 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s2 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: fcvt s0, h0 +; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s1 +; CHECK-GI-NOFP16-NEXT: fcvt h0, s0 +; CHECK-GI-NOFP16-NEXT: ret +; +; CHECK-GI-FP16-LABEL: mul_2H: +; CHECK-GI-FP16: // %bb.0: +; CHECK-GI-FP16-NEXT: fmov h2, #1.00000000 +; CHECK-GI-FP16-NEXT: fmul h2, h2, h0 +; CHECK-GI-FP16-NEXT: fmul h2, h2, v0.h[1] +; CHECK-GI-FP16-NEXT: fmul h2, h2, v0.h[2] +; CHECK-GI-FP16-NEXT: fmul h2, h2, v0.h[3] +; CHECK-GI-FP16-NEXT: fmul h2, h2, v0.h[4] +; CHECK-GI-FP16-NEXT: fmul h2, h2, v0.h[5] +; CHECK-GI-FP16-NEXT: fmul h2, h2, v0.h[6] +; CHECK-GI-FP16-NEXT: fmul h0, h2, v0.h[7] +; CHECK-GI-FP16-NEXT: fmul h0, h0, h1 +; CHECK-GI-FP16-NEXT: fmul h0, h0, v1.h[1] +; CHECK-GI-FP16-NEXT: fmul h0, h0, v1.h[2] +; CHECK-GI-FP16-NEXT: fmul h0, h0, v1.h[3] +; CHECK-GI-FP16-NEXT: fmul h0, h0, v1.h[4] +; CHECK-GI-FP16-NEXT: fmul h0, h0, v1.h[5] +; CHECK-GI-FP16-NEXT: fmul h0, h0, v1.h[6] +; CHECK-GI-FP16-NEXT: fmul h0, h0, v1.h[7] +; CHECK-GI-FP16-NEXT: ret %r = call half @llvm.vector.reduce.fmul.f16.v16f16(half 1.0, <16 x half> %bin.rdx) ret half %r } define float @mul_2S(<8 x float> %bin.rdx) { -; CHECK-LABEL: mul_2S: -; CHECK: // %bb.0: -; CHECK-NEXT: fmul s2, s0, v0.s[1] -; CHECK-NEXT: fmul s2, s2, v0.s[2] -; CHECK-NEXT: fmul s0, s2, v0.s[3] -; CHECK-NEXT: fmul s0, s0, s1 -; CHECK-NEXT: fmul s0, s0, v1.s[1] -; CHECK-NEXT: fmul s0, s0, v1.s[2] -; CHECK-NEXT: fmul s0, s0, v1.s[3] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: mul_2S: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: fmul s2, s0, v0.s[1] +; CHECK-SD-NEXT: fmul s2, s2, v0.s[2] +; CHECK-SD-NEXT: fmul s0, s2, v0.s[3] +; CHECK-SD-NEXT: fmul s0, s0, s1 +; CHECK-SD-NEXT: fmul s0, s0, v1.s[1] +; CHECK-SD-NEXT: fmul s0, s0, v1.s[2] +; CHECK-SD-NEXT: fmul s0, s0, v1.s[3] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: mul_2S: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov s2, #1.00000000 +; CHECK-GI-NEXT: fmul s2, s2, s0 +; CHECK-GI-NEXT: fmul s2, s2, v0.s[1] +; CHECK-GI-NEXT: fmul s2, s2, v0.s[2] +; CHECK-GI-NEXT: fmul s0, s2, v0.s[3] +; CHECK-GI-NEXT: fmul s0, s0, s1 +; CHECK-GI-NEXT: fmul s0, s0, v1.s[1] +; CHECK-GI-NEXT: fmul s0, s0, v1.s[2] +; CHECK-GI-NEXT: fmul s0, s0, v1.s[3] +; CHECK-GI-NEXT: ret %r = call float @llvm.vector.reduce.fmul.f32.v8f32(float 1.0, <8 x float> %bin.rdx) ret float %r } define double @mul_2D(<4 x double> %bin.rdx) { -; CHECK-LABEL: mul_2D: -; CHECK: // %bb.0: -; CHECK-NEXT: fmul d0, d0, v0.d[1] -; CHECK-NEXT: fmul d0, d0, d1 -; CHECK-NEXT: fmul d0, d0, v1.d[1] -; CHECK-NEXT: ret +; CHECK-SD-LABEL: mul_2D: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: fmul d0, d0, v0.d[1] +; CHECK-SD-NEXT: fmul d0, d0, d1 +; CHECK-SD-NEXT: fmul d0, d0, v1.d[1] +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: mul_2D: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: fmov d2, #1.00000000 +; CHECK-GI-NEXT: fmul d2, d2, d0 +; CHECK-GI-NEXT: fmul d0, d2, v0.d[1] +; CHECK-GI-NEXT: fmul d0, d0, d1 +; CHECK-GI-NEXT: fmul d0, d0, v1.d[1] +; CHECK-GI-NEXT: ret %r = call double @llvm.vector.reduce.fmul.f64.v4f64(double 1.0, <4 x double> %bin.rdx) ret double %r } @@ -270,5 +521,3 @@ declare float @llvm.vector.reduce.fmul.f32.v4f32(float, <4 x float>) declare float @llvm.vector.reduce.fmul.f32.v8f32(float, <8 x float>) declare double @llvm.vector.reduce.fmul.f64.v2f64(double, <2 x double>) declare double @llvm.vector.reduce.fmul.f64.v4f64(double, <4 x double>) -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; CHECK-SD: {{.*}} -- GitLab From b5ced67a2a99a2a4a419832192475c516b8b6bb2 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 5 Jan 2024 16:19:21 +0800 Subject: [PATCH 075/728] [CodeGenPassBuilder] Replace `AnalysisKey` -> `MachinePassKey` (#77053) Forgot to update them in #75567. --- llvm/include/llvm/CodeGen/CodeGenPassBuilder.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index 06da9fb57902..a7cbb0910baa 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -229,25 +229,27 @@ protected: C(&PassT::Key); } - template void insertPass(AnalysisKey *ID, PassT Pass) { + template void insertPass(MachinePassKey *ID, PassT Pass) { AfterCallbacks.emplace_back( - [this, ID, Pass = std::move(Pass)](AnalysisKey *PassID) { + [this, ID, Pass = std::move(Pass)](MachinePassKey *PassID) { if (PassID == ID) this->PM.addPass(std::move(Pass)); }); } - void disablePass(AnalysisKey *ID) { + void disablePass(MachinePassKey *ID) { BeforeCallbacks.emplace_back( - [ID](AnalysisKey *PassID) { return PassID != ID; }); + [ID](MachinePassKey *PassID) { return PassID != ID; }); } MachineFunctionPassManager releasePM() { return std::move(PM); } private: MachineFunctionPassManager &PM; - SmallVector, 4> BeforeCallbacks; - SmallVector, 4> AfterCallbacks; + SmallVector, 4> + BeforeCallbacks; + SmallVector, 4> + AfterCallbacks; }; LLVMTargetMachine &TM; -- GitLab From 59f3b7202dc4590215e1fba92e35ec2c0eeb2f65 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 5 Jan 2024 08:19:50 +0000 Subject: [PATCH 076/728] [AMDGPU] Add GXF12 8- and 16-bit SMEM loads (#76966) --- llvm/lib/Target/AMDGPU/SMInstructions.td | 18 + llvm/test/MC/AMDGPU/gfx12_asm_smem.s | 564 ++++++++++++++++++ .../Disassembler/AMDGPU/gfx12_dasm_smem.txt | 513 ++++++++++++++++ 3 files changed, 1095 insertions(+) diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td index be21cf0140fc..087ee65aa03f 100644 --- a/llvm/lib/Target/AMDGPU/SMInstructions.td +++ b/llvm/lib/Target/AMDGPU/SMInstructions.td @@ -305,6 +305,10 @@ let SubtargetPredicate = HasScalarDwordx3Loads in defm S_LOAD_DWORDX4 : SM_Pseudo_Loads ; defm S_LOAD_DWORDX8 : SM_Pseudo_Loads ; defm S_LOAD_DWORDX16 : SM_Pseudo_Loads ; +defm S_LOAD_I8 : SM_Pseudo_Loads ; +defm S_LOAD_U8 : SM_Pseudo_Loads ; +defm S_LOAD_I16 : SM_Pseudo_Loads ; +defm S_LOAD_U16 : SM_Pseudo_Loads ; let is_buffer = 1 in { defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads ; @@ -316,6 +320,10 @@ let SubtargetPredicate = HasScalarDwordx3Loads in defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads ; defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads ; defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads ; +defm S_BUFFER_LOAD_I8 : SM_Pseudo_Loads ; +defm S_BUFFER_LOAD_U8 : SM_Pseudo_Loads ; +defm S_BUFFER_LOAD_I16 : SM_Pseudo_Loads ; +defm S_BUFFER_LOAD_U16 : SM_Pseudo_Loads ; } let SubtargetPredicate = HasScalarStores in { @@ -1336,6 +1344,11 @@ defm S_LOAD_B128 : SM_Real_Loads_gfx12<0x02, "S_LOAD_DWORDX4">; defm S_LOAD_B256 : SM_Real_Loads_gfx12<0x03, "S_LOAD_DWORDX8">; defm S_LOAD_B512 : SM_Real_Loads_gfx12<0x04, "S_LOAD_DWORDX16">; +defm S_LOAD_I8 : SM_Real_Loads_gfx12<0x08>; +defm S_LOAD_U8 : SM_Real_Loads_gfx12<0x09>; +defm S_LOAD_I16 : SM_Real_Loads_gfx12<0x0a>; +defm S_LOAD_U16 : SM_Real_Loads_gfx12<0x0b>; + defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx12<0x10, "S_BUFFER_LOAD_DWORD">; defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx12<0x11, "S_BUFFER_LOAD_DWORDX2">; defm S_BUFFER_LOAD_B96 : SM_Real_Loads_gfx12<0x15, "S_BUFFER_LOAD_DWORDX3">; @@ -1343,6 +1356,11 @@ defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx12<0x12, "S_BUFFER_LOAD_DWORDX4">; defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx12<0x13, "S_BUFFER_LOAD_DWORDX8">; defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx12<0x14, "S_BUFFER_LOAD_DWORDX16">; +defm S_BUFFER_LOAD_I8 : SM_Real_Loads_gfx12<0x18>; +defm S_BUFFER_LOAD_U8 : SM_Real_Loads_gfx12<0x19>; +defm S_BUFFER_LOAD_I16 : SM_Real_Loads_gfx12<0x1a>; +defm S_BUFFER_LOAD_U16 : SM_Real_Loads_gfx12<0x1b>; + def S_DCACHE_INV_gfx12 : SMEM_Real_gfx12<0x021, S_DCACHE_INV>; def S_PREFETCH_INST_gfx12 : SMEM_Real_Prefetch_gfx12<0x24, S_PREFETCH_INST>; diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_smem.s b/llvm/test/MC/AMDGPU/gfx12_asm_smem.s index 1566b9c04e34..eb59607755da 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_smem.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_smem.s @@ -34,6 +34,306 @@ s_buffer_prefetch_data s[20:23], 100, s10, 7 s_buffer_prefetch_data s[20:23], 100, null, 7 // GFX12: s_buffer_prefetch_data s[20:23], 0x64, null, 7 ; encoding: [0xca,0xe1,0x04,0xf4,0x64,0x00,0x00,0xf8] +s_load_i8 s5, s[2:3], s0 +// GFX12: encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s101, s[2:3], s0 +// GFX12: encoding: [0x41,0x19,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 vcc_lo, s[2:3], s0 +// GFX12: encoding: [0x81,0x1a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 vcc_hi, s[2:3], s0 +// GFX12: encoding: [0xc1,0x1a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[4:5], s0 +// GFX12: encoding: [0x42,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[100:101], s0 +// GFX12: encoding: [0x72,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, vcc, s0 +// GFX12: encoding: [0x75,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s101 +// GFX12: encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xca] + +s_load_i8 s5, s[2:3], vcc_lo +// GFX12: encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xd4] + +s_load_i8 s5, s[2:3], vcc_hi +// GFX12: encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xd6] + +s_load_i8 s5, s[2:3], m0 +// GFX12: encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xfa] + +s_load_i8 s5, s[2:3], 0x0 +// GFX12: encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xf8] + +s_load_i8 s5, s[2:3], s7 offset:0x12345 +// GFX12: encoding: [0x41,0x01,0x01,0xf4,0x45,0x23,0x01,0x0e] + +s_load_u8 s5, s[2:3], s0 +// GFX12: encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 +// GFX12: encoding: [0x41,0x39,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 vcc_lo, s[2:3], s0 +// GFX12: encoding: [0x81,0x3a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 vcc_hi, s[2:3], s0 +// GFX12: encoding: [0xc1,0x3a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s5, s[4:5], s0 +// GFX12: encoding: [0x42,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s5, s[100:101], s0 +// GFX12: encoding: [0x72,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s5, vcc, s0 +// GFX12: encoding: [0x75,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s5, s[2:3], s101 +// GFX12: encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xca] + +s_load_u8 s5, s[2:3], vcc_lo +// GFX12: encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xd4] + +s_load_u8 s5, s[2:3], vcc_hi +// GFX12: encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xd6] + +s_load_u8 s5, s[2:3], m0 +// GFX12: encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xfa] + +s_load_u8 s5, s[2:3], 0x0 +// GFX12: encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xf8] + +s_load_u8 s5, s[2:3], s7 offset:0x12345 +// GFX12: encoding: [0x41,0x21,0x01,0xf4,0x45,0x23,0x01,0x0e] + +s_buffer_load_i8 s5, s[4:7], s0 +// GFX12: encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 s101, s[4:7], s0 +// GFX12: encoding: [0x42,0x19,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 +// GFX12: encoding: [0x82,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_hi, s[4:7], s0 +// GFX12: encoding: [0xc2,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 s5, s[8:11], s0 +// GFX12: encoding: [0x44,0x01,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 s5, s[96:99], s0 +// GFX12: encoding: [0x70,0x01,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 s5, s[4:7], s101 +// GFX12: encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xca] + +s_buffer_load_i8 s5, s[4:7], vcc_lo +// GFX12: encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xd4] + +s_buffer_load_i8 s5, s[4:7], vcc_hi +// GFX12: encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xd6] + +s_buffer_load_i8 s5, s[4:7], m0 +// GFX12: encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xfa] + +s_buffer_load_i8 s5, s[4:7], 0x0 +// GFX12: encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xf8] + +s_buffer_load_i8 s5, s[4:7], s0 offset:0x12345 +// GFX12: encoding: [0x42,0x01,0x03,0xf4,0x45,0x23,0x01,0x00] + +s_buffer_load_u8 s5, s[4:7], s0 +// GFX12: encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s101, s[4:7], s0 +// GFX12: encoding: [0x42,0x39,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 vcc_lo, s[4:7], s0 +// GFX12: encoding: [0x82,0x3a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 vcc_hi, s[4:7], s0 +// GFX12: encoding: [0xc2,0x3a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 +// GFX12: encoding: [0x44,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[96:99], s0 +// GFX12: encoding: [0x70,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[4:7], s101 +// GFX12: encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xca] + +s_buffer_load_u8 s5, s[4:7], vcc_lo +// GFX12: encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xd4] + +s_buffer_load_u8 s5, s[4:7], vcc_hi +// GFX12: encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xd6] + +s_buffer_load_u8 s5, s[4:7], m0 +// GFX12: encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xfa] + +s_buffer_load_u8 s5, s[4:7], 0x0 +// GFX12: encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xf8] + +s_buffer_load_u8 s5, s[4:7], s0 offset:0x12345 +// GFX12: encoding: [0x42,0x21,0x03,0xf4,0x45,0x23,0x01,0x00] + +s_load_i16 s5, s[2:3], s0 +// GFX12: encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s101, s[2:3], s0 +// GFX12: encoding: [0x41,0x59,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 vcc_lo, s[2:3], s0 +// GFX12: encoding: [0x81,0x5a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 vcc_hi, s[2:3], s0 +// GFX12: encoding: [0xc1,0x5a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[4:5], s0 +// GFX12: encoding: [0x42,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 +// GFX12: encoding: [0x72,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, vcc, s0 +// GFX12: encoding: [0x75,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[2:3], s101 +// GFX12: encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xca] + +s_load_i16 s5, s[2:3], vcc_lo +// GFX12: encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xd4] + +s_load_i16 s5, s[2:3], vcc_hi +// GFX12: encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xd6] + +s_load_i16 s5, s[2:3], m0 +// GFX12: encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xfa] + +s_load_i16 s5, s[2:3], 0x0 +// GFX12: encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xf8] + +s_load_i16 s5, s[2:3], s7 offset:0x12345 +// GFX12: encoding: [0x41,0x41,0x01,0xf4,0x45,0x23,0x01,0x0e] + +s_load_u16 s5, s[2:3], s0 +// GFX12: encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s101, s[2:3], s0 +// GFX12: encoding: [0x41,0x79,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 vcc_lo, s[2:3], s0 +// GFX12: encoding: [0x81,0x7a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 vcc_hi, s[2:3], s0 +// GFX12: encoding: [0xc1,0x7a,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, s[4:5], s0 +// GFX12: encoding: [0x42,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, s[100:101], s0 +// GFX12: encoding: [0x72,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 +// GFX12: encoding: [0x75,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, s[2:3], s101 +// GFX12: encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xca] + +s_load_u16 s5, s[2:3], vcc_lo +// GFX12: encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xd4] + +s_load_u16 s5, s[2:3], vcc_hi +// GFX12: encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xd6] + +s_load_u16 s5, s[2:3], m0 +// GFX12: encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xfa] + +s_load_u16 s5, s[2:3], 0x0 +// GFX12: encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xf8] + +s_load_u16 s5, s[2:3], s7 offset:0x12345 +// GFX12: encoding: [0x41,0x61,0x01,0xf4,0x45,0x23,0x01,0x0e] + +s_buffer_load_i16 s5, s[4:7], s0 +// GFX12: encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 s101, s[4:7], s0 +// GFX12: encoding: [0x42,0x59,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_lo, s[4:7], s0 +// GFX12: encoding: [0x82,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 +// GFX12: encoding: [0xc2,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 s5, s[8:11], s0 +// GFX12: encoding: [0x44,0x41,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 s5, s[96:99], s0 +// GFX12: encoding: [0x70,0x41,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 s5, s[4:7], s101 +// GFX12: encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xca] + +s_buffer_load_i16 s5, s[4:7], vcc_lo +// GFX12: encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xd4] + +s_buffer_load_i16 s5, s[4:7], vcc_hi +// GFX12: encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xd6] + +s_buffer_load_i16 s5, s[4:7], m0 +// GFX12: encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xfa] + +s_buffer_load_i16 s5, s[4:7], 0x0 +// GFX12: encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xf8] + +s_buffer_load_i16 s5, s[4:7], s0 offset:0x12345 +// GFX12: encoding: [0x42,0x41,0x03,0xf4,0x45,0x23,0x01,0x00] + +s_buffer_load_u16 s5, s[4:7], s0 +// GFX12: encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s101, s[4:7], s0 +// GFX12: encoding: [0x42,0x79,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 vcc_lo, s[4:7], s0 +// GFX12: encoding: [0x82,0x7a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 vcc_hi, s[4:7], s0 +// GFX12: encoding: [0xc2,0x7a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[8:11], s0 +// GFX12: encoding: [0x44,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 +// GFX12: encoding: [0x70,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[4:7], s101 +// GFX12: encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xca] + +s_buffer_load_u16 s5, s[4:7], vcc_lo +// GFX12: encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xd4] + +s_buffer_load_u16 s5, s[4:7], vcc_hi +// GFX12: encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xd6] + +s_buffer_load_u16 s5, s[4:7], m0 +// GFX12: encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xfa] + +s_buffer_load_u16 s5, s[4:7], 0x0 +// GFX12: encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xf8] + +s_buffer_load_u16 s5, s[4:7], s0 offset:0x12345 +// GFX12: encoding: [0x42,0x61,0x03,0xf4,0x45,0x23,0x01,0x00] + s_load_b32 s5, s[2:3], s0 // GFX12: s_load_b32 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x01,0x00,0xf4,0x00,0x00,0x00,0x00] @@ -646,6 +946,138 @@ s_load_b512 s[20:35], s[4:5], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE s_load_b512 s[20:35], s[4:5], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS // GFX12: s_load_b512 s[20:35], s[4:5], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x02,0x85,0x60,0xf5,0x00,0x00,0x00,0x00] +s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x41,0x01,0x81,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x41,0x01,0x01,0xf5,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x41,0x01,0x81,0xf5,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x41,0x01,0x21,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x41,0x01,0x41,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x41,0x01,0x61,0xf4,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x41,0x01,0x21,0xf5,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x41,0x01,0xa1,0xf5,0x00,0x00,0x00,0x00] + +s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x41,0x01,0x61,0xf5,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x39,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x41,0x39,0x81,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x41,0x39,0x01,0xf5,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x41,0x39,0x81,0xf5,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x39,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x41,0x39,0x21,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x41,0x39,0x41,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x41,0x39,0x61,0xf4,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x41,0x39,0x21,0xf5,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x41,0x39,0xa1,0xf5,0x00,0x00,0x00,0x00] + +s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x41,0x39,0x61,0xf5,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 ; encoding: [0x72,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x72,0x41,0x81,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x72,0x41,0x01,0xf5,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x72,0x41,0x81,0xf5,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 ; encoding: [0x72,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x72,0x41,0x21,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x72,0x41,0x41,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x72,0x41,0x61,0xf4,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x72,0x41,0x21,0xf5,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x72,0x41,0xa1,0xf5,0x00,0x00,0x00,0x00] + +s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x72,0x41,0x61,0xf5,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 ; encoding: [0x75,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x75,0x61,0x81,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x75,0x61,0x01,0xf5,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x75,0x61,0x81,0xf5,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 ; encoding: [0x75,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x75,0x61,0x21,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x75,0x61,0x41,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x75,0x61,0x61,0xf4,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x75,0x61,0x21,0xf5,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x75,0x61,0xa1,0xf5,0x00,0x00,0x00,0x00] + +s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x75,0x61,0x61,0xf5,0x00,0x00,0x00,0x00] + s_buffer_load_b32 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_RT // GFX12: s_buffer_load_b32 s5, s[96:99], s0 offset:0x0 ; encoding: [0x70,0x01,0x02,0xf4,0x00,0x00,0x00,0x00] @@ -843,3 +1275,135 @@ s_buffer_load_b512 s[20:35], s[96:99], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_S s_buffer_load_b512 s[20:35], s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS // GFX12: s_buffer_load_b512 s[20:35], s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x30,0x85,0x62,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 ; encoding: [0x82,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x82,0x1a,0x83,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x82,0x1a,0x03,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x82,0x1a,0x83,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 ; encoding: [0x82,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x82,0x1a,0x23,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x82,0x1a,0x43,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x82,0x1a,0x63,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x82,0x1a,0x23,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x82,0x1a,0xa3,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x82,0x1a,0x63,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 ; encoding: [0x44,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x44,0x21,0x83,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x44,0x21,0x03,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x44,0x21,0x83,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 ; encoding: [0x44,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x44,0x21,0x23,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x44,0x21,0x43,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x44,0x21,0x63,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x44,0x21,0x23,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x44,0x21,0xa3,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x44,0x21,0x63,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 ; encoding: [0xc2,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0xc2,0x5a,0x83,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0xc2,0x5a,0x03,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0xc2,0x5a,0x83,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 ; encoding: [0xc2,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0xc2,0x5a,0x23,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0xc2,0x5a,0x43,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0xc2,0x5a,0x63,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0xc2,0x5a,0x23,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0xc2,0x5a,0xa3,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0xc2,0x5a,0x63,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_RT +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 ; encoding: [0x70,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_NT +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x70,0x61,0x83,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x70,0x61,0x03,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_LU +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x70,0x61,0x83,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_CU +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 ; encoding: [0x70,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_SE +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x70,0x61,0x23,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_DEV +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x70,0x61,0x43,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_SYS +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x70,0x61,0x63,0xf4,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x70,0x61,0x23,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x70,0x61,0xa3,0xf5,0x00,0x00,0x00,0x00] + +s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS +// GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x70,0x61,0x63,0xf5,0x00,0x00,0x00,0x00] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt index f46e74537b9c..81cd930d2261 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt @@ -1,5 +1,302 @@ # RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s101, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x19,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x19,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 vcc_lo, s[2:3], s0 offset:0x0 ; encoding: [0x81,0x1a,0x01,0xf4,0x00,0x00,0x00,0x00] +0x81,0x1a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 vcc_hi, s[2:3], s0 offset:0x0 ; encoding: [0xc1,0x1a,0x01,0xf4,0x00,0x00,0x00,0x00] +0xc1,0x1a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[4:5], s0 offset:0x0 ; encoding: [0x42,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] +0x42,0x01,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[100:101], s0 offset:0x0 ; encoding: [0x72,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] +0x72,0x01,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, vcc, s0 offset:0x0 ; encoding: [0x75,0x01,0x01,0xf4,0x00,0x00,0x00,0x00] +0x75,0x01,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s101 offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xca] +0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_load_i8 s5, s[2:3], vcc_lo offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xd4] +0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_load_i8 s5, s[2:3], vcc_hi offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xd6] +0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_load_i8 s5, s[2:3], m0 offset:0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xfa] +0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_load_i8 s5, s[2:3], s7 offset:0x12345 ; encoding: [0x41,0x01,0x01,0xf4,0x45,0x23,0x01,0x0e] +0x41,0x01,0x01,0xf4,0x45,0x23,0x01,0x0e + +# GFX12: s_load_i8 s5, s[2:3], 0x0 ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xf8] +0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0xf8 + +# GFX12: s_load_u8 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x39,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x39,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 vcc_lo, s[2:3], s0 offset:0x0 ; encoding: [0x81,0x3a,0x01,0xf4,0x00,0x00,0x00,0x00] +0x81,0x3a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 vcc_hi, s[2:3], s0 offset:0x0 ; encoding: [0xc1,0x3a,0x01,0xf4,0x00,0x00,0x00,0x00] +0xc1,0x3a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s5, s[4:5], s0 offset:0x0 ; encoding: [0x42,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] +0x42,0x21,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s5, s[100:101], s0 offset:0x0 ; encoding: [0x72,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] +0x72,0x21,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s5, vcc, s0 offset:0x0 ; encoding: [0x75,0x21,0x01,0xf4,0x00,0x00,0x00,0x00] +0x75,0x21,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s5, s[2:3], s101 offset:0x0 ; encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xca] +0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_load_u8 s5, s[2:3], vcc_lo offset:0x0 ; encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xd4] +0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_load_u8 s5, s[2:3], vcc_hi offset:0x0 ; encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xd6] +0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_load_u8 s5, s[2:3], m0 offset:0x0 ; encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xfa] +0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_load_u8 s5, s[2:3], s7 offset:0x12345 ; encoding: [0x41,0x21,0x01,0xf4,0x45,0x23,0x01,0x0e] +0x41,0x21,0x01,0xf4,0x45,0x23,0x01,0x0e + +# GFX12: s_load_u8 s5, s[2:3], 0x0 ; encoding: [0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xf8] +0x41,0x21,0x01,0xf4,0x00,0x00,0x00,0xf8 + +# GFX12: s_buffer_load_i8 s5, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 s101, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x19,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x19,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 ; encoding: [0x82,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00] +0x82,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_hi, s[4:7], s0 offset:0x0 ; encoding: [0xc2,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x1a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 s5, s[8:11], s0 offset:0x0 ; encoding: [0x44,0x01,0x03,0xf4,0x00,0x00,0x00,0x00] +0x44,0x01,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 s5, s[96:99], s0 offset:0x0 ; encoding: [0x70,0x01,0x03,0xf4,0x00,0x00,0x00,0x00] +0x70,0x01,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 s5, s[4:7], s101 offset:0x0 ; encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xca] +0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_buffer_load_i8 s5, s[4:7], vcc_lo offset:0x0 ; encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xd4] +0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_buffer_load_i8 s5, s[4:7], vcc_hi offset:0x0 ; encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xd6] +0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_buffer_load_i8 s5, s[4:7], m0 offset:0x0 ; encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xfa] +0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_buffer_load_i8 s5, s[4:7], 0x0 ; encoding: [0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xf8] +0x42,0x01,0x03,0xf4,0x00,0x00,0x00,0xf8 + +# GFX12: s_buffer_load_i8 s5, s[4:7], s0 offset:0x12345 ; encoding: [0x42,0x01,0x03,0xf4,0x45,0x23,0x01,0x00] +0x42,0x01,0x03,0xf4,0x45,0x23,0x01,0x00 + +# GFX12: s_buffer_load_u8 s5, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s101, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x39,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x39,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 vcc_lo, s[4:7], s0 offset:0x0 ; encoding: [0x82,0x3a,0x03,0xf4,0x00,0x00,0x00,0x00] +0x82,0x3a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 vcc_hi, s[4:7], s0 offset:0x0 ; encoding: [0xc2,0x3a,0x03,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x3a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 ; encoding: [0x44,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] +0x44,0x21,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[96:99], s0 offset:0x0 ; encoding: [0x70,0x21,0x03,0xf4,0x00,0x00,0x00,0x00] +0x70,0x21,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[4:7], s101 offset:0x0 ; encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xca] +0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_buffer_load_u8 s5, s[4:7], vcc_lo offset:0x0 ; encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xd4] +0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_buffer_load_u8 s5, s[4:7], vcc_hi offset:0x0 ; encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xd6] +0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_buffer_load_u8 s5, s[4:7], m0 offset:0x0 ; encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xfa] +0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_buffer_load_u8 s5, s[4:7], 0x0 ; encoding: [0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xf8] +0x42,0x21,0x03,0xf4,0x00,0x00,0x00,0xf8 + +# GFX12: s_buffer_load_u8 s5, s[4:7], s0 offset:0x12345 ; encoding: [0x42,0x21,0x03,0xf4,0x45,0x23,0x01,0x00] +0x42,0x21,0x03,0xf4,0x45,0x23,0x01,0x00 + +# GFX12: s_load_i16 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s101, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x59,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x59,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 vcc_lo, s[2:3], s0 offset:0x0 ; encoding: [0x81,0x5a,0x01,0xf4,0x00,0x00,0x00,0x00] +0x81,0x5a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 vcc_hi, s[2:3], s0 offset:0x0 ; encoding: [0xc1,0x5a,0x01,0xf4,0x00,0x00,0x00,0x00] +0xc1,0x5a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[4:5], s0 offset:0x0 ; encoding: [0x42,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] +0x42,0x41,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 ; encoding: [0x72,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] +0x72,0x41,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, vcc, s0 offset:0x0 ; encoding: [0x75,0x41,0x01,0xf4,0x00,0x00,0x00,0x00] +0x75,0x41,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[2:3], s101 offset:0x0 ; encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xca] +0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_load_i16 s5, s[2:3], vcc_lo offset:0x0 ; encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xd4] +0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_load_i16 s5, s[2:3], vcc_hi offset:0x0 ; encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xd6] +0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_load_i16 s5, s[2:3], m0 offset:0x0 ; encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xfa] +0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_load_i16 s5, s[2:3], s7 offset:0x12345 ; encoding: [0x41,0x41,0x01,0xf4,0x45,0x23,0x01,0x0e] +0x41,0x41,0x01,0xf4,0x45,0x23,0x01,0x0e + +# GFX12: s_load_i16 s5, s[2:3], 0x0 ; encoding: [0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xf8] +0x41,0x41,0x01,0xf4,0x00,0x00,0x00,0xf8 + +# GFX12: s_load_u16 s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s101, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x79,0x01,0xf4,0x00,0x00,0x00,0x00] +0x41,0x79,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 vcc_lo, s[2:3], s0 offset:0x0 ; encoding: [0x81,0x7a,0x01,0xf4,0x00,0x00,0x00,0x00] +0x81,0x7a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 vcc_hi, s[2:3], s0 offset:0x0 ; encoding: [0xc1,0x7a,0x01,0xf4,0x00,0x00,0x00,0x00] +0xc1,0x7a,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, s[4:5], s0 offset:0x0 ; encoding: [0x42,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] +0x42,0x61,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, s[100:101], s0 offset:0x0 ; encoding: [0x72,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] +0x72,0x61,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 ; encoding: [0x75,0x61,0x01,0xf4,0x00,0x00,0x00,0x00] +0x75,0x61,0x01,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, s[2:3], s101 offset:0x0 ; encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xca] +0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_load_u16 s5, s[2:3], vcc_lo offset:0x0 ; encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xd4] +0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_load_u16 s5, s[2:3], vcc_hi offset:0x0 ; encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xd6] +0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_load_u16 s5, s[2:3], m0 offset:0x0 ; encoding: [0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xfa] +0x41,0x61,0x01,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_load_u16 s5, s[2:3], s7 offset:0x12345 ; encoding: [0x41,0x61,0x01,0xf4,0x45,0x23,0x01,0x0e] +0x41,0x61,0x01,0xf4,0x45,0x23,0x01,0x0e + +# GFX12: s_buffer_load_i16 s5, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 s101, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x59,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x59,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_lo, s[4:7], s0 offset:0x0 ; encoding: [0x82,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00] +0x82,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 ; encoding: [0xc2,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 s5, s[8:11], s0 offset:0x0 ; encoding: [0x44,0x41,0x03,0xf4,0x00,0x00,0x00,0x00] +0x44,0x41,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 s5, s[96:99], s0 offset:0x0 ; encoding: [0x70,0x41,0x03,0xf4,0x00,0x00,0x00,0x00] +0x70,0x41,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 s5, s[4:7], s101 offset:0x0 ; encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xca] +0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_buffer_load_i16 s5, s[4:7], vcc_lo offset:0x0 ; encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xd4] +0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_buffer_load_i16 s5, s[4:7], vcc_hi offset:0x0 ; encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xd6] +0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_buffer_load_i16 s5, s[4:7], m0 offset:0x0 ; encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xfa] +0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_buffer_load_i16 s5, s[4:7], 0x0 ; encoding: [0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xf8] +0x42,0x41,0x03,0xf4,0x00,0x00,0x00,0xf8 + +# GFX12: s_buffer_load_i16 s5, s[4:7], s0 offset:0x12345 ; encoding: [0x42,0x41,0x03,0xf4,0x45,0x23,0x01,0x00] +0x42,0x41,0x03,0xf4,0x45,0x23,0x01,0x00 + +# GFX12: s_buffer_load_u16 s5, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s101, s[4:7], s0 offset:0x0 ; encoding: [0x42,0x79,0x03,0xf4,0x00,0x00,0x00,0x00] +0x42,0x79,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 vcc_lo, s[4:7], s0 offset:0x0 ; encoding: [0x82,0x7a,0x03,0xf4,0x00,0x00,0x00,0x00] +0x82,0x7a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 vcc_hi, s[4:7], s0 offset:0x0 ; encoding: [0xc2,0x7a,0x03,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x7a,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[8:11], s0 offset:0x0 ; encoding: [0x44,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] +0x44,0x61,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 ; encoding: [0x70,0x61,0x03,0xf4,0x00,0x00,0x00,0x00] +0x70,0x61,0x03,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[4:7], s101 offset:0x0 ; encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xca] +0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xca + +# GFX12: s_buffer_load_u16 s5, s[4:7], vcc_lo offset:0x0 ; encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xd4] +0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xd4 + +# GFX12: s_buffer_load_u16 s5, s[4:7], vcc_hi offset:0x0 ; encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xd6] +0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xd6 + +# GFX12: s_buffer_load_u16 s5, s[4:7], m0 offset:0x0 ; encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xfa] +0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xfa + +# GFX12: s_buffer_load_u16 s5, s[4:7], 0x0 ; encoding: [0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xf8] +0x42,0x61,0x03,0xf4,0x00,0x00,0x00,0xf8 + +# GFX12: s_buffer_load_u16 s5, s[4:7], s0 offset:0x12345 ; encoding: [0x42,0x61,0x03,0xf4,0x45,0x23,0x01,0x00] +0x42,0x61,0x03,0xf4,0x45,0x23,0x01,0x00 + # GFX12: s_load_b32 s101, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x19,0x00,0xf4,0x00,0x00,0x00,0x00] 0x41,0x19,0x00,0xf4,0x00,0x00,0x00,0x00 @@ -603,6 +900,114 @@ # GFX12: s_load_b512 s[20:35], s[4:5], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x02,0x85,0x60,0xf5,0x00,0x00,0x00,0x00] 0x02,0x85,0x60,0xf5,0x00,0x00,0x00,0x00 +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x41,0x01,0x81,0xf4,0x00,0x00,0x00,0x00] +0x41,0x01,0x81,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x41,0x01,0x01,0xf5,0x00,0x00,0x00,0x00] +0x41,0x01,0x01,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x41,0x01,0x81,0xf5,0x00,0x00,0x00,0x00] +0x41,0x01,0x81,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x41,0x01,0x21,0xf4,0x00,0x00,0x00,0x00] +0x41,0x01,0x21,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x41,0x01,0x41,0xf4,0x00,0x00,0x00,0x00] +0x41,0x01,0x41,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x41,0x01,0x61,0xf4,0x00,0x00,0x00,0x00] +0x41,0x01,0x61,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x41,0x01,0x21,0xf5,0x00,0x00,0x00,0x00] +0x41,0x01,0x21,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x41,0x01,0xa1,0xf5,0x00,0x00,0x00,0x00] +0x41,0x01,0xa1,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x41,0x01,0x61,0xf5,0x00,0x00,0x00,0x00] +0x41,0x01,0x61,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x41,0x39,0x81,0xf4,0x00,0x00,0x00,0x00] +0x41,0x39,0x81,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x41,0x39,0x01,0xf5,0x00,0x00,0x00,0x00] +0x41,0x39,0x01,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x41,0x39,0x81,0xf5,0x00,0x00,0x00,0x00] +0x41,0x39,0x81,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x41,0x39,0x21,0xf4,0x00,0x00,0x00,0x00] +0x41,0x39,0x21,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x41,0x39,0x41,0xf4,0x00,0x00,0x00,0x00] +0x41,0x39,0x41,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x41,0x39,0x61,0xf4,0x00,0x00,0x00,0x00] +0x41,0x39,0x61,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x41,0x39,0x21,0xf5,0x00,0x00,0x00,0x00] +0x41,0x39,0x21,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x41,0x39,0xa1,0xf5,0x00,0x00,0x00,0x00] +0x41,0x39,0xa1,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u8 s101, s[2:3], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x41,0x39,0x61,0xf5,0x00,0x00,0x00,0x00] +0x41,0x39,0x61,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x72,0x41,0x81,0xf4,0x00,0x00,0x00,0x00] +0x72,0x41,0x81,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x72,0x41,0x01,0xf5,0x00,0x00,0x00,0x00] +0x72,0x41,0x01,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x72,0x41,0x81,0xf5,0x00,0x00,0x00,0x00] +0x72,0x41,0x81,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x72,0x41,0x21,0xf4,0x00,0x00,0x00,0x00] +0x72,0x41,0x21,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x72,0x41,0x41,0xf4,0x00,0x00,0x00,0x00] +0x72,0x41,0x41,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x72,0x41,0x61,0xf4,0x00,0x00,0x00,0x00] +0x72,0x41,0x61,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x72,0x41,0x21,0xf5,0x00,0x00,0x00,0x00] +0x72,0x41,0x21,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x72,0x41,0xa1,0xf5,0x00,0x00,0x00,0x00] +0x72,0x41,0xa1,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_i16 s5, s[100:101], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x72,0x41,0x61,0xf5,0x00,0x00,0x00,0x00] +0x72,0x41,0x61,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x75,0x61,0x81,0xf4,0x00,0x00,0x00,0x00] +0x75,0x61,0x81,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x75,0x61,0x01,0xf5,0x00,0x00,0x00,0x00] +0x75,0x61,0x01,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x75,0x61,0x81,0xf5,0x00,0x00,0x00,0x00] +0x75,0x61,0x81,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x75,0x61,0x21,0xf4,0x00,0x00,0x00,0x00] +0x75,0x61,0x21,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x75,0x61,0x41,0xf4,0x00,0x00,0x00,0x00] +0x75,0x61,0x41,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x75,0x61,0x61,0xf4,0x00,0x00,0x00,0x00] +0x75,0x61,0x61,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x75,0x61,0x21,0xf5,0x00,0x00,0x00,0x00] +0x75,0x61,0x21,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x75,0x61,0xa1,0xf5,0x00,0x00,0x00,0x00] +0x75,0x61,0xa1,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_load_u16 s5, vcc, s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x75,0x61,0x61,0xf5,0x00,0x00,0x00,0x00] +0x75,0x61,0x61,0xf5,0x00,0x00,0x00,0x00 + # GFX12: s_buffer_load_b32 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x70,0x01,0x82,0xf4,0x00,0x00,0x00,0x00] 0x70,0x01,0x82,0xf4,0x00,0x00,0x00,0x00 @@ -764,3 +1169,111 @@ # GFX12: s_buffer_load_b512 s[20:35], s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x30,0x85,0x62,0xf5,0x00,0x00,0x00,0x00] 0x30,0x85,0x62,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x82,0x1a,0x83,0xf4,0x00,0x00,0x00,0x00] +0x82,0x1a,0x83,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x82,0x1a,0x03,0xf5,0x00,0x00,0x00,0x00] +0x82,0x1a,0x03,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x82,0x1a,0x83,0xf5,0x00,0x00,0x00,0x00] +0x82,0x1a,0x83,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x82,0x1a,0x23,0xf4,0x00,0x00,0x00,0x00] +0x82,0x1a,0x23,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x82,0x1a,0x43,0xf4,0x00,0x00,0x00,0x00] +0x82,0x1a,0x43,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x82,0x1a,0x63,0xf4,0x00,0x00,0x00,0x00] +0x82,0x1a,0x63,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x82,0x1a,0x23,0xf5,0x00,0x00,0x00,0x00] +0x82,0x1a,0x23,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x82,0x1a,0xa3,0xf5,0x00,0x00,0x00,0x00] +0x82,0x1a,0xa3,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i8 vcc_lo, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x82,0x1a,0x63,0xf5,0x00,0x00,0x00,0x00] +0x82,0x1a,0x63,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x44,0x21,0x83,0xf4,0x00,0x00,0x00,0x00] +0x44,0x21,0x83,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x44,0x21,0x03,0xf5,0x00,0x00,0x00,0x00] +0x44,0x21,0x03,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x44,0x21,0x83,0xf5,0x00,0x00,0x00,0x00] +0x44,0x21,0x83,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x44,0x21,0x23,0xf4,0x00,0x00,0x00,0x00] +0x44,0x21,0x23,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x44,0x21,0x43,0xf4,0x00,0x00,0x00,0x00] +0x44,0x21,0x43,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x44,0x21,0x63,0xf4,0x00,0x00,0x00,0x00] +0x44,0x21,0x63,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x44,0x21,0x23,0xf5,0x00,0x00,0x00,0x00] +0x44,0x21,0x23,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x44,0x21,0xa3,0xf5,0x00,0x00,0x00,0x00] +0x44,0x21,0xa3,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u8 s5, s[8:11], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x44,0x21,0x63,0xf5,0x00,0x00,0x00,0x00] +0x44,0x21,0x63,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0xc2,0x5a,0x83,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x83,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0xc2,0x5a,0x03,0xf5,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x03,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0xc2,0x5a,0x83,0xf5,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x83,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0xc2,0x5a,0x23,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x23,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0xc2,0x5a,0x43,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x43,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0xc2,0x5a,0x63,0xf4,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x63,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0xc2,0x5a,0x23,0xf5,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x23,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0xc2,0x5a,0xa3,0xf5,0x00,0x00,0x00,0x00] +0xc2,0x5a,0xa3,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_i16 vcc_hi, s[4:7], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0xc2,0x5a,0x63,0xf5,0x00,0x00,0x00,0x00] +0xc2,0x5a,0x63,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_NT ; encoding: [0x70,0x61,0x83,0xf4,0x00,0x00,0x00,0x00] +0x70,0x61,0x83,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT ; encoding: [0x70,0x61,0x03,0xf5,0x00,0x00,0x00,0x00] +0x70,0x61,0x03,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_LU ; encoding: [0x70,0x61,0x83,0xf5,0x00,0x00,0x00,0x00] +0x70,0x61,0x83,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_SE ; encoding: [0x70,0x61,0x23,0xf4,0x00,0x00,0x00,0x00] +0x70,0x61,0x23,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_DEV ; encoding: [0x70,0x61,0x43,0xf4,0x00,0x00,0x00,0x00] +0x70,0x61,0x43,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 scope:SCOPE_SYS ; encoding: [0x70,0x61,0x63,0xf4,0x00,0x00,0x00,0x00] +0x70,0x61,0x63,0xf4,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SE ; encoding: [0x70,0x61,0x23,0xf5,0x00,0x00,0x00,0x00] +0x70,0x61,0x23,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_LU scope:SCOPE_SE ; encoding: [0x70,0x61,0xa3,0xf5,0x00,0x00,0x00,0x00] +0x70,0x61,0xa3,0xf5,0x00,0x00,0x00,0x00 + +# GFX12: s_buffer_load_u16 s5, s[96:99], s0 offset:0x0 th:TH_LOAD_HT scope:SCOPE_SYS ; encoding: [0x70,0x61,0x63,0xf5,0x00,0x00,0x00,0x00] +0x70,0x61,0x63,0xf5,0x00,0x00,0x00,0x00 -- GitLab From 4d7c5ad58467502fcbc433591edff40d8a4d697d Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 5 Jan 2024 16:20:08 +0800 Subject: [PATCH 077/728] [NewPM] Update CodeGenPreparePass reference in CodeGenPassBuilder (#77054) Follows #75380. --- llvm/include/llvm/CodeGen/CodeGenPassBuilder.h | 3 ++- llvm/include/llvm/CodeGen/MachinePassRegistry.def | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index a7cbb0910baa..fc3da3491412 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -24,6 +24,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/CallBrPrepare.h" +#include "llvm/CodeGen/CodeGenPrepare.h" #include "llvm/CodeGen/DwarfEHPrepare.h" #include "llvm/CodeGen/ExpandMemCmp.h" #include "llvm/CodeGen/ExpandReductions.h" @@ -729,7 +730,7 @@ void CodeGenPassBuilder::addPassesToHandleExceptions( template void CodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const { if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP) - addPass(CodeGenPreparePass()); + addPass(CodeGenPreparePass(&TM)); // TODO: Default ctor'd RewriteSymbolPass is no-op. // addPass(RewriteSymbolPass()); } diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index f950dfae7e33..bf88a43b8a0c 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -42,6 +42,7 @@ FUNCTION_ANALYSIS("targetir", TargetIRAnalysis, #endif FUNCTION_PASS("callbrprepare", CallBrPreparePass, ()) FUNCTION_PASS("cfguard", CFGuardPass, ()) +FUNCTION_PASS("codegenprepare", CodeGenPreparePass, (TM)) FUNCTION_PASS("consthoist", ConstantHoistingPass, ()) FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass, (TM)) FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false)) @@ -131,7 +132,6 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) #endif DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ()) -DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ()) DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ()) DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ()) -- GitLab From e96e7a9a864b39a69455d5baad271f04cbf2d909 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 5 Jan 2024 08:20:52 +0000 Subject: [PATCH 078/728] [AMDGPU] Implement readcyclecounter for GFX12 (#76965) --- llvm/lib/Target/AMDGPU/AMDGPU.td | 10 ++++- llvm/lib/Target/AMDGPU/GCNSubtarget.h | 5 +++ llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 42 ++++++++++++++++++++ llvm/lib/Target/AMDGPU/SIInstructions.td | 6 +++ llvm/test/CodeGen/AMDGPU/readcyclecounter.ll | 12 ++++++ 5 files changed, 74 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index d2a325d5ad89..df8c35ffd457 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -761,6 +761,12 @@ def FeatureShaderCyclesRegister : SubtargetFeature<"shader-cycles-register", "Has SHADER_CYCLES hardware register" >; +def FeatureShaderCyclesHiLoRegisters : SubtargetFeature<"shader-cycles-hi-lo-registers", + "HasShaderCyclesHiLoRegisters", + "true", + "Has SHADER_CYCLES_HI/LO hardware registers" +>; + def FeatureMadMacF32Insts : SubtargetFeature<"mad-mac-f32-insts", "HasMadMacF32Insts", "true", @@ -1469,7 +1475,7 @@ def FeatureISAVersion12 : FeatureSet< FeatureNSAEncoding, FeaturePartialNSAEncoding, FeatureWavefrontSize32, - FeatureShaderCyclesRegister, + FeatureShaderCyclesHiLoRegisters, FeatureArchitectedFlatScratch, FeatureAtomicFaddRtnInsts, FeatureAtomicFaddNoRtnInsts, @@ -1970,6 +1976,8 @@ def HasSMemTimeInst : Predicate<"Subtarget->hasSMemTimeInst()">, def HasShaderCyclesRegister : Predicate<"Subtarget->hasShaderCyclesRegister()">, AssemblerPredicate<(all_of FeatureShaderCyclesRegister)>; +def HasShaderCyclesHiLoRegisters : Predicate<"Subtarget->hasShaderCyclesHiLoRegisters()">; + def HasFP8Insts : Predicate<"Subtarget->hasFP8Insts()">, AssemblerPredicate<(all_of FeatureFP8Insts)>; diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index b85eb76aca26..ce3164d7b92e 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -176,6 +176,7 @@ protected: bool HasGetWaveIdInst = false; bool HasSMemTimeInst = false; bool HasShaderCyclesRegister = false; + bool HasShaderCyclesHiLoRegisters = false; bool HasVOP3Literal = false; bool HasNoDataDepHazard = false; bool FlatAddressSpace = false; @@ -819,6 +820,10 @@ public: return HasShaderCyclesRegister; } + bool hasShaderCyclesHiLoRegisters() const { + return HasShaderCyclesHiLoRegisters; + } + bool hasVOP3Literal() const { return HasVOP3Literal; } diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index c84a0934ca81..079cae06888c 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -4909,6 +4909,48 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter( MI.eraseFromParent(); return BB; } + case AMDGPU::GET_SHADERCYCLESHILO: { + assert(MF->getSubtarget().hasShaderCyclesHiLoRegisters()); + MachineRegisterInfo &MRI = MF->getRegInfo(); + const DebugLoc &DL = MI.getDebugLoc(); + // The algorithm is: + // + // hi1 = getreg(SHADER_CYCLES_HI) + // lo1 = getreg(SHADER_CYCLES_LO) + // hi2 = getreg(SHADER_CYCLES_HI) + // + // If hi1 == hi2 then there was no overflow and the result is hi2:lo1. + // Otherwise there was overflow and the result is hi2:0. In both cases the + // result should represent the actual time at some point during the sequence + // of three getregs. + Register RegHi1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegHi1) + .addImm(AMDGPU::Hwreg::encodeHwreg(AMDGPU::Hwreg::ID_SHADER_CYCLES_HI, + 0, 32)); + Register RegLo1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegLo1) + .addImm( + AMDGPU::Hwreg::encodeHwreg(AMDGPU::Hwreg::ID_SHADER_CYCLES, 0, 32)); + Register RegHi2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegHi2) + .addImm(AMDGPU::Hwreg::encodeHwreg(AMDGPU::Hwreg::ID_SHADER_CYCLES_HI, + 0, 32)); + BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CMP_EQ_U32)) + .addReg(RegHi1) + .addReg(RegHi2); + Register RegLo = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CSELECT_B32), RegLo) + .addReg(RegLo1) + .addImm(0); + BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE)) + .add(MI.getOperand(0)) + .addReg(RegLo) + .addImm(AMDGPU::sub0) + .addReg(RegHi2) + .addImm(AMDGPU::sub1); + MI.eraseFromParent(); + return BB; + } case AMDGPU::SI_INDIRECT_SRC_V1: case AMDGPU::SI_INDIRECT_SRC_V2: case AMDGPU::SI_INDIRECT_SRC_V4: diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 1158d16baa1b..ea2a8b75d074 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -316,6 +316,12 @@ def S_USUBO_PSEUDO : SPseudoInstSI < (outs SReg_32:$sdst, SSrc_i1:$scc_out), (ins SSrc_b32:$src0, SSrc_b32:$src1) >; +let OtherPredicates = [HasShaderCyclesHiLoRegisters] in +def GET_SHADERCYCLESHILO : SPseudoInstSI< + (outs SReg_64:$sdst), (ins), + [(set SReg_64:$sdst, (i64 (readcyclecounter)))] +>; + } // End usesCustomInserter = 1, Defs = [SCC] let usesCustomInserter = 1 in { diff --git a/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll b/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll index 7e0a486c8191..17b3fdc04ec9 100644 --- a/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll +++ b/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll @@ -8,12 +8,19 @@ ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GETREG,GETREG-GISEL -check-prefix=GCN %s ; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GETREG,GETREG-SDAG -check-prefix=GCN %s ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GETREG,GETREG-GISEL -check-prefix=GCN %s +; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX12 %s +; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX12 %s declare i64 @llvm.readcyclecounter() #0 ; GCN-LABEL: {{^}}test_readcyclecounter: ; MEMTIME-DAG: s_memtime s{{\[[0-9]+:[0-9]+\]}} ; GCN-DAG: s_load_{{dwordx2|b64}} +; GFX12: s_getreg_b32 [[HI1:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_HI) +; GFX12: s_getreg_b32 [[LO1:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_LO) +; GFX12: s_getreg_b32 [[HI2:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_HI) +; GFX12: s_cmp_eq_u32 [[HI1]], [[HI2]] +; GFX12: s_cselect_b32 {{s[0-9]+}}, [[LO1]], 0 ; GCN-DAG: lgkmcnt ; MEMTIME: store_dwordx2 ; SIVI-NOT: lgkmcnt @@ -43,8 +50,13 @@ define amdgpu_kernel void @test_readcyclecounter(ptr addrspace(1) %out) #0 { ; ; GCN-LABEL: {{^}}test_readcyclecounter_smem: ; MEMTIME-DAG: s_memtime +; GFX12: s_getreg_b32 [[HI1:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_HI) +; GFX12: s_getreg_b32 [[LO1:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_LO) +; GFX12: s_getreg_b32 [[HI2:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_HI) ; GCN-DAG: s_load_{{dword|b32|b64}} ; GETREG-DAG: s_getreg_b32 s{{[0-9]+}}, hwreg(HW_REG_SHADER_CYCLES, 0, 20) +; GFX12: s_cmp_eq_u32 [[HI1]], [[HI2]] +; GFX12: s_cselect_b32 {{s[0-9]+}}, [[LO1]], 0 define amdgpu_cs i32 @test_readcyclecounter_smem(ptr addrspace(4) inreg %in) #0 { %cycle0 = call i64 @llvm.readcyclecounter() %in.v = load i64, ptr addrspace(4) %in -- GitLab From bb6d5c220004a5d7e466a669324001285a688918 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Fri, 5 Jan 2024 09:22:18 +0100 Subject: [PATCH 079/728] [mlir][Transforms] `GreedyPatternRewriteDriver`: Do not CSE constants during iterations (#75897) The `GreedyPatternRewriteDriver` tries to iteratively fold ops and apply rewrite patterns to ops. It has special handling for constants: they are CSE'd and sometimes moved to parent regions to allow for additional CSE'ing. This happens in `OperationFolder`. To allow for efficient CSE'ing, `OperationFolder` maintains an internal lookup data structure to find the existing constant ops with the same value for each `IsolatedFromAbove` region: ```c++ /// A mapping between an insertion region and the constants that have been /// created within it. DenseMap foldScopes; ``` Rewrite patterns are allowed to modify operations. In particular, they may move operations (including constants) from one region to another one. Such an IR rewrite can make the above lookup data structure inconsistent. We encountered such a bug in a downstream project. This bug materialized in the form of an op that uses the result of a constant op from a different `IsolatedFromAbove` region (that is not accessible). This commit changes the behavior of the `GreedyPatternRewriteDriver` such that `OperationFolder` is used to CSE constants at the beginning of each iteration (as the worklist is populated), but no longer during an iteration. `OperationFolder` is no longer used after populating the worklist, so we do not have to care about inconsistent state in the `OperationFolder` due to IR rewrites. The `GreedyPatternRewriteDriver` now performs the op folding by itself instead of calling `OperationFolder::tryToFold`. This change changes the order of constant ops in test cases, but not the region in which they appear. All broken test cases were fixed by turning `CHECK` into `CHECK-DAG`. Alternatives considered: The state of `OperationFolder` could be partially invalidated with every `notifyOperationModified` notification. That is more fragile than the solution in this commit because incorrect rewriter API usage can lead to missing notifications and hard-to-debug `IsolatedFromAbove` violations. (It did not fix the above mention bug in a downstream project, which could be due to incorrect rewriter API usage or due to another conceptual problem that I missed.) Moreover, ops are frequently getting modified during a greedy pattern rewrite, so we would likely keep invalidating large parts of the state of `OperationFolder` over and over. Migration guide: Turn `CHECK` into `CHECK-DAG` in test cases. Constant ops are no longer folded during a greedy pattern rewrite. If you rely on folding (and rematerialization) of constant ops during a greedy pattern rewrite, turn the folder into a pattern. --- flang/test/Lower/array-temp.f90 | 54 +++++------ .../Linalg/Transforms/DecomposeLinalgOps.cpp | 5 + .../Utils/GreedyPatternRewriteDriver.cpp | 84 +++++++++++----- .../VectorToArmSME/vector-to-arm-sme.mlir | 10 +- .../VectorToLLVM/vector-to-llvm.mlir | 8 +- .../Conversion/VectorToSCF/vector-to-scf.mlir | 30 +++--- .../test/Dialect/ArmSME/arith-ops-to-sme.mlir | 16 ++-- .../test/Dialect/LLVMIR/type-consistency.mlir | 58 +++++------ mlir/test/Dialect/Linalg/loops.mlir | 16 ++-- .../Linalg/vectorization-with-patterns.mlir | 6 +- .../Linalg/vectorize-tensor-extract.mlir | 96 +++++++++---------- .../Math/algebraic-simplification.mlir | 28 +++--- mlir/test/Dialect/Math/expand-math.mlir | 24 ++--- .../Math/polynomial-approximation.mlir | 72 +++++++------- .../Dialect/Mesh/resharding-spmdization.mlir | 8 +- .../NVGPU/transform-create-async-groups.mlir | 12 +-- mlir/test/Dialect/SCF/loop-pipelining.mlir | 6 +- mlir/test/Dialect/SparseTensor/sparse_1d.mlir | 8 +- .../Dialect/SparseTensor/sparse_affine.mlir | 8 +- .../Dialect/SparseTensor/sparse_concat.mlir | 8 +- .../Dialect/SparseTensor/sparse_storage.mlir | 8 +- .../Tensor/fold-into-pack-and-unpack.mlir | 16 ++-- mlir/test/Dialect/Tosa/constant-op-fold.mlir | 6 +- .../Tosa/tosa-decompose-depthwise.mlir | 8 +- .../Tosa/tosa-decompose-transpose-conv.mlir | 15 ++- .../vector-broadcast-lowering-transforms.mlir | 4 +- ...tract-to-matrix-intrinsics-transforms.mlir | 6 +- .../vector-mask-lowering-transforms.mlir | 14 +-- .../vector-multi-reduction-lowering.mlir | 34 +++---- .../vector-scalable-create-mask-lowering.mlir | 4 +- 30 files changed, 357 insertions(+), 315 deletions(-) diff --git a/flang/test/Lower/array-temp.f90 b/flang/test/Lower/array-temp.f90 index 971b3506fbe3..347d4cef78bc 100644 --- a/flang/test/Lower/array-temp.f90 +++ b/flang/test/Lower/array-temp.f90 @@ -43,15 +43,15 @@ end ! CHECK-LABEL: func @_QPss2( ! CHECK-SAME: %arg0: !fir.ref {fir.bindc_name = "n"}) { -! CHECK: %[[C_m1:[-0-9a-z_]+]] = arith.constant -1 : index -! CHECK: %[[C_2:[-0-9a-z_]+]] = arith.constant 2 : index -! CHECK: %[[C_1:[-0-9a-z_]+]] = arith.constant 1 : index -! CHECK: %[[C_27_i32:[-0-9a-z_]+]] = arith.constant 27 : i32 -! CHECK: %[[C_6_i32:[-0-9a-z_]+]] = arith.constant 6 : i32 -! CHECK: %[[C_st:[-0-9a-z_]+]] = arith.constant 7.000000e+00 : f32 -! CHECK: %[[C_1_i32:[-0-9a-z_]+]] = arith.constant 1 : i32 -! CHECK: %[[C_st_0:[-0-9a-z_]+]] = arith.constant -2.000000e+00 : f32 -! CHECK: %[[C_0:[-0-9a-z_]+]] = arith.constant 0 : index +! CHECK-DAG: %[[C_m1:[-0-9a-z_]+]] = arith.constant -1 : index +! CHECK-DAG: %[[C_2:[-0-9a-z_]+]] = arith.constant 2 : index +! CHECK-DAG: %[[C_1:[-0-9a-z_]+]] = arith.constant 1 : index +! CHECK-DAG: %[[C_27_i32:[-0-9a-z_]+]] = arith.constant 27 : i32 +! CHECK-DAG: %[[C_6_i32:[-0-9a-z_]+]] = arith.constant 6 : i32 +! CHECK-DAG: %[[C_st:[-0-9a-z_]+]] = arith.constant 7.000000e+00 : f32 +! CHECK-DAG: %[[C_1_i32:[-0-9a-z_]+]] = arith.constant 1 : i32 +! CHECK-DAG: %[[C_st_0:[-0-9a-z_]+]] = arith.constant -2.000000e+00 : f32 +! CHECK-DAG: %[[C_0:[-0-9a-z_]+]] = arith.constant 0 : index ! CHECK: %[[V_0:[0-9]+]] = fir.load %arg0 : !fir.ref ! CHECK: %[[V_1:[0-9]+]] = fir.convert %[[V_0:[0-9]+]] : (i32) -> index ! CHECK: %[[V_2:[0-9]+]] = arith.cmpi sgt, %[[V_1]], %[[C_0]] : index @@ -137,15 +137,15 @@ end ! CHECK-LABEL: func @_QPss3( ! CHECK-SAME: %arg0: !fir.ref {fir.bindc_name = "n"}) { -! CHECK: %[[C_m1:[-0-9a-z_]+]] = arith.constant -1 : index -! CHECK: %[[C_2:[-0-9a-z_]+]] = arith.constant 2 : index -! CHECK: %[[C_1:[-0-9a-z_]+]] = arith.constant 1 : index -! CHECK: %[[C_34_i32:[-0-9a-z_]+]] = arith.constant 34 : i32 -! CHECK: %[[C_6_i32:[-0-9a-z_]+]] = arith.constant 6 : i32 -! CHECK: %[[C_st:[-0-9a-z_]+]] = arith.constant 7.000000e+00 : f32 -! CHECK: %[[C_1_i32:[-0-9a-z_]+]] = arith.constant 1 : i32 -! CHECK: %[[C_st_0:[-0-9a-z_]+]] = arith.constant -2.000000e+00 : f32 -! CHECK: %[[C_0:[-0-9a-z_]+]] = arith.constant 0 : index +! CHECK-DAG: %[[C_m1:[-0-9a-z_]+]] = arith.constant -1 : index +! CHECK-DAG: %[[C_2:[-0-9a-z_]+]] = arith.constant 2 : index +! CHECK-DAG: %[[C_1:[-0-9a-z_]+]] = arith.constant 1 : index +! CHECK-DAG: %[[C_34_i32:[-0-9a-z_]+]] = arith.constant 34 : i32 +! CHECK-DAG: %[[C_6_i32:[-0-9a-z_]+]] = arith.constant 6 : i32 +! CHECK-DAG: %[[C_st:[-0-9a-z_]+]] = arith.constant 7.000000e+00 : f32 +! CHECK-DAG: %[[C_1_i32:[-0-9a-z_]+]] = arith.constant 1 : i32 +! CHECK-DAG: %[[C_st_0:[-0-9a-z_]+]] = arith.constant -2.000000e+00 : f32 +! CHECK-DAG: %[[C_0:[-0-9a-z_]+]] = arith.constant 0 : index ! CHECK: %[[V_0:[0-9]+]] = fir.load %arg0 : !fir.ref ! CHECK: %[[V_1:[0-9]+]] = fir.convert %[[V_0:[0-9]+]] : (i32) -> index ! CHECK: %[[V_2:[0-9]+]] = arith.cmpi sgt, %[[V_1]], %[[C_0]] : index @@ -263,15 +263,15 @@ end ! CHECK-LABEL: func @_QPss4( ! CHECK-SAME: %arg0: !fir.ref {fir.bindc_name = "n"}) { -! CHECK: %[[C_2:[-0-9a-z_]+]] = arith.constant 2 : index -! CHECK: %[[C_m1:[-0-9a-z_]+]] = arith.constant -1 : index -! CHECK: %[[C_1:[-0-9a-z_]+]] = arith.constant 1 : index -! CHECK: %[[C_41_i32:[-0-9a-z_]+]] = arith.constant 41 : i32 -! CHECK: %[[C_6_i32:[-0-9a-z_]+]] = arith.constant 6 : i32 -! CHECK: %[[C_st:[-0-9a-z_]+]] = arith.constant 7.000000e+00 : f32 -! CHECK: %[[C_1_i32:[-0-9a-z_]+]] = arith.constant 1 : i32 -! CHECK: %[[C_st_0:[-0-9a-z_]+]] = arith.constant -2.000000e+00 : f32 -! CHECK: %[[C_0:[-0-9a-z_]+]] = arith.constant 0 : index +! CHECK-DAG: %[[C_2:[-0-9a-z_]+]] = arith.constant 2 : index +! CHECK-DAG: %[[C_m1:[-0-9a-z_]+]] = arith.constant -1 : index +! CHECK-DAG: %[[C_1:[-0-9a-z_]+]] = arith.constant 1 : index +! CHECK-DAG: %[[C_41_i32:[-0-9a-z_]+]] = arith.constant 41 : i32 +! CHECK-DAG: %[[C_6_i32:[-0-9a-z_]+]] = arith.constant 6 : i32 +! CHECK-DAG: %[[C_st:[-0-9a-z_]+]] = arith.constant 7.000000e+00 : f32 +! CHECK-DAG: %[[C_1_i32:[-0-9a-z_]+]] = arith.constant 1 : i32 +! CHECK-DAG: %[[C_st_0:[-0-9a-z_]+]] = arith.constant -2.000000e+00 : f32 +! CHECK-DAG: %[[C_0:[-0-9a-z_]+]] = arith.constant 0 : index ! CHECK: %[[V_0:[0-9]+]] = fir.load %arg0 : !fir.ref ! CHECK: %[[V_1:[0-9]+]] = fir.convert %[[V_0:[0-9]+]] : (i32) -> index ! CHECK: %[[V_2:[0-9]+]] = arith.cmpi sgt, %[[V_1]], %[[C_0]] : index diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp index 1227478118fb..28f4d8ac6443 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @@ -312,6 +312,11 @@ DecomposeLinalgOp::matchAndRewrite(GenericOp genericOp, if (origYield.getDefiningOp() == peeledScalarOperation) { yieldedVals.push_back(origYield); } else { + // Do not materialize any new ops inside of the decomposed LinalgOp, + // as that would trigger another application of the rewrite pattern + // (infinite loop). + OpBuilder::InsertionGuard g(rewriter); + rewriter.setInsertionPoint(peeledGenericOp); yieldedVals.push_back( getZero(rewriter, genericOp.getLoc(), origYield.getType())); } diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp index eca13f52f53d..82438e2bf706 100644 --- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp +++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp @@ -314,9 +314,6 @@ protected: Worklist worklist; #endif // MLIR_GREEDY_REWRITE_RANDOMIZER_SEED - /// Non-pattern based folder for operations. - OperationFolder folder; - /// Configuration information for how to simplify. const GreedyRewriteConfig config; @@ -358,7 +355,7 @@ private: GreedyPatternRewriteDriver::GreedyPatternRewriteDriver( MLIRContext *ctx, const FrozenRewritePatternSet &patterns, const GreedyRewriteConfig &config) - : PatternRewriter(ctx), folder(ctx, this), config(config), matcher(patterns) + : PatternRewriter(ctx), config(config), matcher(patterns) #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS // clang-format off , debugFingerPrints(this) @@ -429,15 +426,55 @@ bool GreedyPatternRewriteDriver::processWorklist() { continue; } - // Try to fold this op. - if (succeeded(folder.tryToFold(op))) { - LLVM_DEBUG(logResultWithLine("success", "operation was folded")); - changed = true; + // Try to fold this op. Do not fold constant ops. That would lead to an + // infinite folding loop, as every constant op would be folded to an + // Attribute and then immediately be rematerialized as a constant op, which + // is then put on the worklist. + if (!op->hasTrait()) { + SmallVector foldResults; + if (succeeded(op->fold(foldResults))) { + LLVM_DEBUG(logResultWithLine("success", "operation was folded")); + changed = true; + if (foldResults.empty()) { + // Op was modified in-place. + notifyOperationModified(op); #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS - if (config.scope && failed(verify(config.scope->getParentOp()))) - llvm::report_fatal_error("IR failed to verify after folding"); + if (config.scope && failed(verify(config.scope->getParentOp()))) + llvm::report_fatal_error("IR failed to verify after folding"); #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS - continue; + continue; + } + + // Op results can be replaced with `foldResults`. + assert(foldResults.size() == op->getNumResults() && + "folder produced incorrect number of results"); + OpBuilder::InsertionGuard g(*this); + setInsertionPoint(op); + SmallVector replacements; + for (auto [ofr, resultType] : + llvm::zip_equal(foldResults, op->getResultTypes())) { + if (auto value = ofr.dyn_cast()) { + assert(value.getType() == resultType && + "folder produced value of incorrect type"); + replacements.push_back(value); + continue; + } + // Materialize Attributes as SSA values. + Operation *constOp = op->getDialect()->materializeConstant( + *this, ofr.get(), resultType, op->getLoc()); + assert(constOp->hasTrait() && + "materializeConstant produced op that is not a ConstantLike"); + assert(constOp->getResultTypes()[0] == resultType && + "materializeConstant produced incorrect result type"); + replacements.push_back(constOp->getResult(0)); + } + replaceOp(op, replacements); +#if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS + if (config.scope && failed(verify(config.scope->getParentOp()))) + llvm::report_fatal_error("IR failed to verify after folding"); +#endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS + continue; + } } // Try to match one of the patterns. The rewriter is automatically @@ -592,7 +629,6 @@ void GreedyPatternRewriteDriver::notifyOperationRemoved(Operation *op) { addOperandsToWorklist(op->getOperands()); worklist.remove(op); - folder.notifyRemoval(op); if (config.strictMode != GreedyRewriteStrictness::AnyOp) strictModeFilteredOps.erase(op); @@ -672,16 +708,6 @@ private: } // namespace LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && { - auto insertKnownConstant = [&](Operation *op) { - // Check for existing constants when populating the worklist. This avoids - // accidentally reversing the constant order during processing. - Attribute constValue; - if (matchPattern(op, m_Constant(&constValue))) - if (!folder.insertKnownConstant(op, constValue)) - return true; - return false; - }; - bool continueRewrites = false; int64_t iteration = 0; MLIRContext *ctx = getContext(); @@ -691,8 +717,22 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && { config.maxIterations != GreedyRewriteConfig::kNoLimit) break; + // New iteration: start with an empty worklist. worklist.clear(); + // `OperationFolder` CSE's constant ops (and may move them into parents + // regions to enable more aggressive CSE'ing). + OperationFolder folder(getContext(), this); + auto insertKnownConstant = [&](Operation *op) { + // Check for existing constants when populating the worklist. This avoids + // accidentally reversing the constant order during processing. + Attribute constValue; + if (matchPattern(op, m_Constant(&constValue))) + if (!folder.insertKnownConstant(op, constValue)) + return true; + return false; + }; + if (!config.useTopDownTraversal) { // Add operations to the worklist in postorder. region.walk([&](Operation *op) { diff --git a/mlir/test/Conversion/VectorToArmSME/vector-to-arm-sme.mlir b/mlir/test/Conversion/VectorToArmSME/vector-to-arm-sme.mlir index 6783263c1849..d3f02c6288a2 100644 --- a/mlir/test/Conversion/VectorToArmSME/vector-to-arm-sme.mlir +++ b/mlir/test/Conversion/VectorToArmSME/vector-to-arm-sme.mlir @@ -309,9 +309,9 @@ func.func @transfer_write_2d_transpose_with_mask_bf16(%vector : vector<[8]x[8]xb // CHECK-LABEL: func.func @broadcast_vec2d_from_i32( // CHECK-SAME: %[[SRC:.*]]: i32) { -// CHECK: %[[C1:.*]] = arith.constant 1 : index -// CHECK: %[[C4:.*]] = arith.constant 4 : index -// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index // CHECK: %[[SRC_1D:.*]] = vector.broadcast %[[SRC]] : i32 to vector<[4]xi32> // CHECK: %[[INIT_TILE:.*]] = arm_sme.get_tile : vector<[4]x[4]xi32> // CHECK: %[[VSCALE:.*]] = vector.vscale @@ -393,8 +393,8 @@ func.func @splat_vec2d_from_f16(%arg0: f16) { // CHECK-LABEL: func.func @transpose_i8( // CHECK-SAME: %[[TILE:.*]]: vector<[16]x[16]xi8>) -// CHECK: %[[C16:.*]] = arith.constant 16 : index -// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C16:.*]] = arith.constant 16 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index // CHECK: %[[VSCALE:.*]] = vector.vscale // CHECK: %[[MIN_TILE_SLICES:.*]] = arith.muli %[[VSCALE]], %[[C16]] : index // CHECK: %[[NUM_TILE_SLICES:.*]] = memref.alloca(%[[MIN_TILE_SLICES]], %[[MIN_TILE_SLICES]]) : memref diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir index 7ea0197bdecb..b13c266609ae 100644 --- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir +++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir @@ -196,10 +196,10 @@ func.func @broadcast_vec3d_from_vec1d(%arg0: vector<2xf32>) -> vector<4x3x2xf32> } // CHECK-LABEL: @broadcast_vec3d_from_vec1d( // CHECK-SAME: %[[A:.*]]: vector<2xf32>) -// CHECK: %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32> -// CHECK: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x2xf32> to !llvm.array<3 x vector<2xf32>> -// CHECK: %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32> -// CHECK: %[[T6:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<4x3x2xf32> to !llvm.array<4 x array<3 x vector<2xf32>>> +// CHECK-DAG: %[[T0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32> +// CHECK-DAG: %[[T2:.*]] = builtin.unrealized_conversion_cast %[[T0]] : vector<3x2xf32> to !llvm.array<3 x vector<2xf32>> +// CHECK-DAG: %[[T1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32> +// CHECK-DAG: %[[T6:.*]] = builtin.unrealized_conversion_cast %[[T1]] : vector<4x3x2xf32> to !llvm.array<4 x array<3 x vector<2xf32>>> // CHECK: %[[T3:.*]] = llvm.insertvalue %[[A]], %[[T2]][0] : !llvm.array<3 x vector<2xf32>> // CHECK: %[[T4:.*]] = llvm.insertvalue %[[A]], %[[T3]][1] : !llvm.array<3 x vector<2xf32>> diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir index 8316b4005cc1..e1babdd2f1f6 100644 --- a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir +++ b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir @@ -533,9 +533,9 @@ func.func @transfer_write_scalable(%arg0: memref> } // CHECK-SAME: %[[ARG_0:.*]]: memref>, -// CHECK: %[[C_0:.*]] = arith.constant 0 : index -// CHECK: %[[C_16:.*]] = arith.constant 16 : index -// CHECK: %[[STEP:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C_0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C_16:.*]] = arith.constant 16 : index +// CHECK-DAG: %[[STEP:.*]] = arith.constant 1 : index // CHECK: %[[MASK_VEC:.*]] = arith.cmpi slt, %{{.*}}, %{{.*}} : vector<[16]xi32> // CHECK: %[[VSCALE:.*]] = vector.vscale // CHECK: %[[UB:.*]] = arith.muli %[[VSCALE]], %[[C_16]] : index @@ -556,8 +556,8 @@ func.func @vector_print_vector_0d(%arg0: vector) { } // CHECK-LABEL: func.func @vector_print_vector_0d( // CHECK-SAME: %[[VEC:.*]]: vector) { -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK: %[[FLAT_VEC:.*]] = vector.shape_cast %[[VEC]] : vector to vector<1xf32> // CHECK: vector.print punctuation // CHECK: scf.for %[[IDX:.*]] = %[[C0]] to %[[C1]] step %[[C1]] { @@ -581,9 +581,9 @@ func.func @vector_print_vector(%arg0: vector<2x2xf32>) { } // CHECK-LABEL: func.func @vector_print_vector( // CHECK-SAME: %[[VEC:.*]]: vector<2x2xf32>) { -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C2:.*]] = arith.constant 2 : index -// CHECK: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK: %[[FLAT_VEC:.*]] = vector.shape_cast %[[VEC]] : vector<2x2xf32> to vector<4xf32> // CHECK: vector.print punctuation // CHECK: scf.for %[[I:.*]] = %[[C0]] to %[[C2]] step %[[C1]] { @@ -650,10 +650,10 @@ func.func @transfer_read_array_of_scalable(%arg0: memref<3x?xf32>) -> vector<3x[ } // CHECK-LABEL: func.func @transfer_read_array_of_scalable( // CHECK-SAME: %[[ARG:.*]]: memref<3x?xf32>) -> vector<3x[4]xf32> { -// CHECK: %[[PADDING:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C3:.*]] = arith.constant 3 : index -// CHECK: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[PADDING:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK: %[[ALLOCA_VEC:.*]] = memref.alloca() : memref> // CHECK: %[[ALLOCA_MASK:.*]] = memref.alloca() : memref> // CHECK: %[[DIM_SIZE:.*]] = memref.dim %[[ARG]], %[[C1]] : memref<3x?xf32> @@ -684,9 +684,9 @@ func.func @transfer_write_array_of_scalable(%vec: vector<3x[4]xf32>, %arg0: memr // CHECK-LABEL: func.func @transfer_write_array_of_scalable( // CHECK-SAME: %[[VEC:.*]]: vector<3x[4]xf32>, // CHECK-SAME: %[[MEMREF:.*]]: memref<3x?xf32>) { -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C3:.*]] = arith.constant 3 : index -// CHECK: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK: %[[ALLOCA_VEC:.*]] = memref.alloca() : memref> // CHECK: %[[ALLOCA_MASK:.*]] = memref.alloca() : memref> // CHECK: %[[DIM_SIZE:.*]] = memref.dim %[[MEMREF]], %[[C1]] : memref<3x?xf32> diff --git a/mlir/test/Dialect/ArmSME/arith-ops-to-sme.mlir b/mlir/test/Dialect/ArmSME/arith-ops-to-sme.mlir index ae2d0f40f03a..e51f2485dadb 100644 --- a/mlir/test/Dialect/ArmSME/arith-ops-to-sme.mlir +++ b/mlir/test/Dialect/ArmSME/arith-ops-to-sme.mlir @@ -91,10 +91,10 @@ func.func @arith_constant_dense_2d_zero_f64() { // ----- // CHECK-LABEL: func.func @arith_constant_dense_2d_nonzero_i8() { -// CHECK: %[[C2_SPLAT:.*]] = arith.constant dense<2> : vector<[16]xi8> -// CHECK: %[[C1:.*]] = arith.constant 1 : index -// CHECK: %[[C16:.*]] = arith.constant 16 : index -// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C2_SPLAT:.*]] = arith.constant dense<2> : vector<[16]xi8> +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C16:.*]] = arith.constant 16 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index // CHECK: %[[INIT_TILE:.*]] = arm_sme.get_tile : vector<[16]x[16]xi8> // CHECK: %[[VSCALE:.*]] = vector.vscale // CHECK: %[[NUM_TILE_SLICES:.*]] = arith.muli %[[VSCALE]], %[[C16]] : index @@ -111,10 +111,10 @@ func.func @arith_constant_dense_2d_nonzero_i8() { // ----- // CHECK-LABEL: func.func @arith_constant_dense_2d_nonzero_f64() { -// CHECK: %[[C2_SPLAT:.*]] = arith.constant dense<2.000000e+00> : vector<[2]xf64> -// CHECK: %[[C1:.*]] = arith.constant 1 : index -// CHECK: %[[C2:.*]] = arith.constant 2 : index -// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C2_SPLAT:.*]] = arith.constant dense<2.000000e+00> : vector<[2]xf64> +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index // CHECK: %[[INIT_TILE:.*]] = arm_sme.get_tile : vector<[2]x[2]xf64> // CHECK: %[[VSCALE:.*]] = vector.vscale // CHECK: %[[NUM_TILE_SLICES:.*]] = arith.muli %[[VSCALE]], %[[C2]] : index diff --git a/mlir/test/Dialect/LLVMIR/type-consistency.mlir b/mlir/test/Dialect/LLVMIR/type-consistency.mlir index 3a1ab924ebda..021151b929d8 100644 --- a/mlir/test/Dialect/LLVMIR/type-consistency.mlir +++ b/mlir/test/Dialect/LLVMIR/type-consistency.mlir @@ -168,8 +168,8 @@ llvm.func @no_crash_on_negative_gep_index() { // CHECK-LABEL: llvm.func @coalesced_store_ints // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_ints(%arg: i64) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32)> @@ -193,8 +193,8 @@ llvm.func @coalesced_store_ints(%arg: i64) { // CHECK-LABEL: llvm.func @coalesced_store_ints_offset // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_ints_offset(%arg: i64) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i64, i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i64, i32, i32)> : (i32) -> !llvm.ptr @@ -218,8 +218,8 @@ llvm.func @coalesced_store_ints_offset(%arg: i64) { // CHECK-LABEL: llvm.func @coalesced_store_floats // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_floats(%arg: i64) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (f32, f32)> @@ -292,9 +292,9 @@ llvm.func @coalesced_store_past_end(%arg: i64) { // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_packed_struct(%arg: i64) { %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST16:.*]] = llvm.mlir.constant(16 : i64) : i64 - // CHECK: %[[CST48:.*]] = llvm.mlir.constant(48 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST16:.*]] = llvm.mlir.constant(16 : i64) : i64 + // CHECK-DAG: %[[CST48:.*]] = llvm.mlir.constant(48 : i64) : i64 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", packed (i16, i32, i16)> %1 = llvm.alloca %0 x !llvm.struct<"foo", packed (i16, i32, i16)> : (i32) -> !llvm.ptr @@ -320,10 +320,10 @@ llvm.func @coalesced_store_packed_struct(%arg: i64) { // CHECK-LABEL: llvm.func @vector_write_split // CHECK-SAME: %[[ARG:.*]]: vector<4xi32> llvm.func @vector_write_split(%arg: vector<4xi32>) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i32) : i32 - // CHECK: %[[CST1:.*]] = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[CST2:.*]] = llvm.mlir.constant(2 : i32) : i32 - // CHECK: %[[CST3:.*]] = llvm.mlir.constant(3 : i32) : i32 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i32) : i32 + // CHECK-DAG: %[[CST1:.*]] = llvm.mlir.constant(1 : i32) : i32 + // CHECK-DAG: %[[CST2:.*]] = llvm.mlir.constant(2 : i32) : i32 + // CHECK-DAG: %[[CST3:.*]] = llvm.mlir.constant(3 : i32) : i32 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32, i32)> : (i32) -> !llvm.ptr @@ -354,10 +354,10 @@ llvm.func @vector_write_split(%arg: vector<4xi32>) { // CHECK-LABEL: llvm.func @vector_write_split_offset // CHECK-SAME: %[[ARG:.*]]: vector<4xi32> llvm.func @vector_write_split_offset(%arg: vector<4xi32>) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i32) : i32 - // CHECK: %[[CST1:.*]] = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[CST2:.*]] = llvm.mlir.constant(2 : i32) : i32 - // CHECK: %[[CST3:.*]] = llvm.mlir.constant(3 : i32) : i32 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i32) : i32 + // CHECK-DAG: %[[CST1:.*]] = llvm.mlir.constant(1 : i32) : i32 + // CHECK-DAG: %[[CST2:.*]] = llvm.mlir.constant(2 : i32) : i32 + // CHECK-DAG: %[[CST3:.*]] = llvm.mlir.constant(3 : i32) : i32 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i64, i32, i32, i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i64, i32, i32, i32, i32)> : (i32) -> !llvm.ptr @@ -470,8 +470,8 @@ llvm.func @gep_split(%arg: i64) { // CHECK-LABEL: llvm.func @coalesced_store_ints_subaggregate // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_ints_subaggregate(%arg: i64) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i64, struct<(i32, i32)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i64, struct<(i32, i32)>)> : (i32) -> !llvm.ptr @@ -513,8 +513,8 @@ llvm.func @gep_result_ptr_type_dynamic(%arg: i64) { // CHECK-LABEL: llvm.func @overlapping_int_aggregate_store // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @overlapping_int_aggregate_store(%arg: i64) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST16:.*]] = llvm.mlir.constant(16 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST16:.*]] = llvm.mlir.constant(16 : i64) : i64 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> @@ -548,10 +548,10 @@ llvm.func @overlapping_int_aggregate_store(%arg: i64) { // CHECK-LABEL: llvm.func @overlapping_vector_aggregate_store // CHECK-SAME: %[[ARG:.*]]: vector<4xi16> llvm.func @overlapping_vector_aggregate_store(%arg: vector<4 x i16>) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i32) : i32 - // CHECK: %[[CST1:.*]] = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[CST2:.*]] = llvm.mlir.constant(2 : i32) : i32 - // CHECK: %[[CST3:.*]] = llvm.mlir.constant(3 : i32) : i32 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i32) : i32 + // CHECK-DAG: %[[CST1:.*]] = llvm.mlir.constant(1 : i32) : i32 + // CHECK-DAG: %[[CST2:.*]] = llvm.mlir.constant(2 : i32) : i32 + // CHECK-DAG: %[[CST3:.*]] = llvm.mlir.constant(3 : i32) : i32 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> @@ -586,8 +586,8 @@ llvm.func @overlapping_vector_aggregate_store(%arg: vector<4 x i16>) { // CHECK-LABEL: llvm.func @partially_overlapping_aggregate_store // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @partially_overlapping_aggregate_store(%arg: i64) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST16:.*]] = llvm.mlir.constant(16 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST16:.*]] = llvm.mlir.constant(16 : i64) : i64 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> @@ -644,8 +644,8 @@ llvm.func @undesirable_overlapping_aggregate_store(%arg: i64) { // CHECK-LABEL: llvm.func @coalesced_store_ints_array // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_ints_array(%arg: i64) { - // CHECK: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 - // CHECK: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 + // CHECK-DAG: %[[CST0:.*]] = llvm.mlir.constant(0 : i64) : i64 + // CHECK-DAG: %[[CST32:.*]] = llvm.mlir.constant(32 : i64) : i64 %0 = llvm.mlir.constant(1 : i32) : i32 // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.array<2 x i32> diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir index 8c13422fd638..b818170a8e79 100644 --- a/mlir/test/Dialect/Linalg/loops.mlir +++ b/mlir/test/Dialect/Linalg/loops.mlir @@ -693,8 +693,8 @@ func.func @conv1d_no_symbols(%in : memref, %filter : memref, %out // CHECK-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref // CHECK-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref // CHECK-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref -// CHECK: %[[c0:.*]] = arith.constant 0 : index -// CHECK: %[[c1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index // CHECK: %[[dim0:.*]] = memref.dim %[[arg1]], %[[c0]] : memref // CHECK: %[[dim1:.*]] = memref.dim %[[arg2]], %[[c0]] : memref // CHECK: scf.for %[[b:.*]] = %[[c0]] to %[[dim1]] step %[[c1]] { @@ -711,8 +711,8 @@ func.func @conv1d_no_symbols(%in : memref, %filter : memref, %out // CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref // CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref // CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref -// CHECKPARALLEL: %[[c0:.*]] = arith.constant 0 : index -// CHECKPARALLEL: %[[c1:.*]] = arith.constant 1 : index +// CHECKPARALLEL-DAG: %[[c0:.*]] = arith.constant 0 : index +// CHECKPARALLEL-DAG: %[[c1:.*]] = arith.constant 1 : index // CHECKPARALLEL: %[[dim0:.*]] = memref.dim %[[arg1]], %[[c0]] : memref // CHECKPARALLEL: %[[dim1:.*]] = memref.dim %[[arg2]], %[[c0]] : memref // CHECKPARALLEL: scf.parallel (%[[b:.*]]) = (%[[c0]]) to (%[[dim1]]) step (%[[c1]]) { @@ -735,8 +735,8 @@ func.func @conv2d_no_symbols(%in : memref, %filter : memref, % // CHECK-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref // CHECK-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref // CHECK-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref -// CHECK: %[[c0:.*]] = arith.constant 0 : index -// CHECK: %[[c1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index // CHECK: %[[dim0:.*]] = memref.dim %[[arg1]], %[[c0]] : memref // CHECK: %[[dim1:.*]] = memref.dim %[[arg1]], %[[c1]] : memref // CHECK: %[[dim2:.*]] = memref.dim %[[arg2]], %[[c0]] : memref @@ -760,8 +760,8 @@ func.func @conv2d_no_symbols(%in : memref, %filter : memref, % // CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref // CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref // CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref -// CHECKPARALLEL: %[[c0:.*]] = arith.constant 0 : index -// CHECKPARALLEL: %[[c1:.*]] = arith.constant 1 : index +// CHECKPARALLEL-DAG: %[[c0:.*]] = arith.constant 0 : index +// CHECKPARALLEL-DAG: %[[c1:.*]] = arith.constant 1 : index // CHECKPARALLEL: %[[dim0:.*]] = memref.dim %[[arg1]], %[[c0]] : memref // CHECKPARALLEL: %[[dim1:.*]] = memref.dim %[[arg1]], %[[c1]] : memref // CHECKPARALLEL: %[[dim2:.*]] = memref.dim %[[arg2]], %[[c0]] : memref diff --git a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir index a43cc2df88ca..58d4b21ea2dd 100644 --- a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir +++ b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir @@ -335,9 +335,9 @@ func.func @vectorize_affine_apply(%arg0: tensor<5xf32>, %arg3: index) -> tensor< // CHECK-LABEL: func.func @vectorize_affine_apply // CHECK-SAME: %arg0: tensor<5xf32> // CHECK-SAME: %[[ARG1:.*]]: index -// CHECK: %[[CST:.*]] = arith.constant dense<[123, 124, 125, 126, 127]> : vector<5xindex> -// CHECK: %[[CST_0:.*]] = arith.constant dense<1> : vector<5xindex> -// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[CST:.*]] = arith.constant dense<[123, 124, 125, 126, 127]> : vector<5xindex> +// CHECK-DAG: %[[CST_0:.*]] = arith.constant dense<1> : vector<5xindex> +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index // CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<5xi32> // CHECK: %[[BCAST:.*]] = vector.broadcast %[[ARG1]] : index to vector<5xindex> // CHECK: %[[ADDI_1:.*]] = arith.addi %[[BCAST]], %[[CST]] : vector<5xindex> diff --git a/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir b/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir index 3fd4fcd53662..96953c234a08 100644 --- a/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir +++ b/mlir/test/Dialect/Linalg/vectorize-tensor-extract.mlir @@ -17,9 +17,9 @@ func.func @vectorize_1d_tensor_extract(%arg0: tensor<3xf32>, %arg1: tensor<4x3xi // CHECK-LABEL: func.func @vectorize_1d_tensor_extract // CHECK-SAME: %[[ARG0:.*]]: tensor<3xf32> // CHECK-SAME: %[[ARG1:.*]]: tensor<4x3xi32> -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[MASK:.*]] = arith.constant dense : vector<4x7x3x2xi1> -// CHECK: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32> +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[MASK:.*]] = arith.constant dense : vector<4x7x3x2xi1> +// CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32> // CHECK: %[[V0:.*]] = vector.transfer_read %[[ARG1]] // CHECK: %[[CAST:.*]] = arith.index_cast %[[V0]] // CHECK: %[[BROADCAST:.*]] = vector.broadcast %[[CAST]] @@ -58,7 +58,7 @@ func.func @vectorize_nd_tensor_extract_constant_idx(%arg0: tensor<3x3xf32>, %arg // CHECK-SAME: %[[ARG_1:.*]]: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> { // CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index -// CHECK-DAG: arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[C0_f32_2:.*]] = arith.constant 0.000000e+00 : f32 // CHECK-DAG: %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32 // CHECK: %[[READ:.*]] = vector.transfer_read %[[ARG_0]][%[[C1]], %[[C2]]], %[[C0_f32]] {in_bounds = [true, true, true], permutation_map = #[[$MAP]]} : tensor<3x3xf32>, vector<1x1x3xf32> // CHECK: %[[C0_4:.*]] = arith.constant 0 : index @@ -93,10 +93,10 @@ func.func @vectorize_nd_tensor_extract_transfer_read_basic(%arg0: tensor<3x3x3xf // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_transfer_read_basic // CHECK-SAME: %[[ARG0:.*]]: tensor<3x3x3xf32> // CHECK-SAME: %[[ARG1:.*]]: tensor<1x1x3xf32> -// CHECK: %[[CST:.*]] = arith.constant dense<0> : vector<1x1x3xindex> -// CHECK: %[[C0_i32:.*]] = arith.constant 0 : i32 -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[CST:.*]] = arith.constant dense<0> : vector<1x1x3xindex> +// CHECK-DAG: %[[C0_i32:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32 // CHECK: %[[IDX_VEC0:.*]] = vector.shape_cast %[[CST]] : vector<1x1x3xindex> to vector<3xindex> // CHECK: %[[IDX1:.*]] = vector.extractelement %[[IDX_VEC0]][%[[C0_i32]] : i32] : vector<3xindex> // CHECK: %[[IDX_VEC:.*]] = vector.shape_cast %[[CST]] : vector<1x1x3xindex> to vector<3xindex> @@ -139,11 +139,11 @@ func.func @vectorize_nd_tensor_extract_transfer_read_complex(%6: tensor<45x80x16 // CHECK-SAME: %[[VAL_0:.*]]: tensor<45x80x16xf32>, // CHECK-SAME: %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: index, %[[VAL_4:.*]]: index, // CHECK-SAME: %[[VAL_5:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { -// CHECK: %[[VAL_6:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> -// CHECK: %[[VAL_7:.*]] = arith.constant 0 : i32 -// CHECK: %[[VAL_8:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[VAL_9:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_10:.*]] = arith.constant 79 : index +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> +// CHECK-DAG: %[[VAL_7:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[VAL_8:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[VAL_9:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_10:.*]] = arith.constant 79 : index // CHECK: %[[VAL_11:.*]] = arith.addi %[[VAL_1]], %[[VAL_2]] : index // CHECK: %[[VAL_12:.*]] = vector.broadcast %[[VAL_11]] : index to vector<1x4xindex> // CHECK: %[[VAL_13:.*]] = vector.broadcast %[[VAL_3]] : index to vector<4xindex> @@ -191,11 +191,11 @@ func.func @vectorize_nd_tensor_extract_index_from_tensor(%arg0: tensor<3x3xf32>, // CHECK-SAME: %[[ARG2:arg2]]: tensor<4x3xi32> // CHECK-SAME: %[[ARG3:.*]]: tensor<4x7x2xf32> // CHECK-SAME: %[[ARG4:.*]]: tensor<4x7x3x2xf32> -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C0_i32:.*]] = arith.constant 0 : i32 -// CHECK: %[[CST:.*]] = arith.constant dense<3> : vector<7x2x4x3xindex> -// CHECK: %[[CST_1:.*]] = arith.constant dense : vector<4x7x3x2xi1> -// CHECK: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32> +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C0_i32:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[CST:.*]] = arith.constant dense<3> : vector<7x2x4x3xindex> +// CHECK-DAG: %[[CST_1:.*]] = arith.constant dense : vector<4x7x3x2xi1> +// CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32> // CHECK: %[[V0:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], %[[C0_i32]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32> // CHECK: %[[V1:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], %[[C0_i32]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32> // CHECK: %[[CAST:.*]] = arith.index_cast %[[V0]] : vector<4x3xi32> to vector<4x3xindex> @@ -239,12 +239,12 @@ func.func @vectorize_nd_tensor_extract_contiguous_and_gather(%arg0: tensor<6xf32 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_contiguous_and_gather( // CHECK-SAME: %[[VAL_0:.*]]: tensor<6xf32> // CHECK-SAME: %[[VAL_1:.*]]: tensor<5xi32> -// CHECK: %[[VAL_2:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_3:.*]] = arith.constant 0 : i32 -// CHECK: %[[VAL_4:.*]] = arith.constant dense<0> : vector<5xindex> -// CHECK: %[[VAL_5:.*]] = arith.constant dense<5> : vector<5xindex> -// CHECK: %[[VAL_6:.*]] = arith.constant dense : vector<5xi1> -// CHECK: %[[VAL_7:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32> +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<0> : vector<5xindex> +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<5> : vector<5xindex> +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant dense : vector<5xi1> +// CHECK-DAG: %[[VAL_7:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32> // CHECK: %[[VAL_8:.*]] = tensor.empty() : tensor<5xf32> // CHECK: %[[VAL_9:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[VAL_3]] {in_bounds = [true]} : tensor<5xi32>, vector<5xi32> // CHECK: %[[VAL_10:.*]] = arith.index_cast %[[VAL_9]] : vector<5xi32> to vector<5xindex> @@ -285,11 +285,11 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(%6: tensor<8 // CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>, // CHECK-SAME: %[[VAL_1:.*]]: index, // CHECK-SAME: %[[VAL_2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { -// CHECK: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> -// CHECK: %[[VAL_4:.*]] = arith.constant 0 : i32 -// CHECK: %[[VAL_5:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[VAL_6:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_7:.*]] = arith.constant 79 : index +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_7:.*]] = arith.constant 79 : index // CHECK: %[[VAL_8:.*]] = vector.broadcast %[[VAL_1]] : index to vector<4xindex> // CHECK: %[[VAL_9:.*]] = arith.addi %[[VAL_8]], %[[VAL_3]] : vector<4xindex> // CHECK: %[[VAL_10:.*]] = vector.extractelement %[[VAL_9]]{{\[}}%[[VAL_4]] : i32] : vector<4xindex> @@ -373,11 +373,11 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_gather(%6: tensor<80x16 // CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>, // CHECK-SAME: %[[VAL_1:.*]]: index, // CHECK-SAME: %[[VAL_2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { -// CHECK: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> -// CHECK: %[[VAL_4:.*]] = arith.constant dense : vector<1x4xi1> -// CHECK: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32> -// CHECK: %[[VAL_6:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_7:.*]] = arith.constant dense<16> : vector<1x4xindex> +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense : vector<1x4xi1> +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32> +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_7:.*]] = arith.constant dense<16> : vector<1x4xindex> // CHECK: %[[VAL_8:.*]] = vector.broadcast %[[VAL_1]] : index to vector<4xindex> // CHECK: %[[VAL_9:.*]] = arith.addi %[[VAL_8]], %[[VAL_3]] : vector<4xindex> // CHECK: %[[VAL_10:.*]] = vector.broadcast %[[VAL_9]] : vector<4xindex> to vector<1x4xindex> @@ -418,11 +418,11 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_gather(%arg0: tensor<80x16xf32 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_maxsi_gather( // CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>, // CHECK-SAME: %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { -// CHECK: %[[VAL_2:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> -// CHECK: %[[VAL_3:.*]] = arith.constant dense<1264> : vector<1x4xindex> -// CHECK: %[[VAL_4:.*]] = arith.constant dense : vector<1x4xi1> -// CHECK: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32> -// CHECK: %[[VAL_6:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<1264> : vector<1x4xindex> +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense : vector<1x4xi1> +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32> +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0 : index // CHECK: %[[VAL_7:.*]] = vector.broadcast %[[VAL_2]] : vector<4xindex> to vector<1x4xindex> // CHECK: %[[VAL_8:.*]] = arith.addi %[[VAL_7]], %[[VAL_3]] : vector<1x4xindex> // CHECK: %[[VAL_9:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {{\[}}%[[VAL_8]]], %[[VAL_4]], %[[VAL_5]] : tensor<80x16xf32>, vector<1x4xindex>, vector<1x4xi1>, vector<1x4xf32> into vector<1x4xf32> @@ -461,10 +461,10 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(%arg0: tensor<80x16 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous( // CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>, // CHECK-SAME: %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { -// CHECK: %[[VAL_2:.*]] = arith.constant dense<16> : vector<1x4xindex> -// CHECK: %[[VAL_3:.*]] = arith.constant 0 : i32 -// CHECK: %[[VAL_4:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_5:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant dense<16> : vector<1x4xindex> +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 0.000000e+00 : f32 // CHECK: %[[VAL_6:.*]] = vector.shape_cast %[[VAL_2]] : vector<1x4xindex> to vector<4xindex> // CHECK: %[[VAL_7:.*]] = vector.extractelement %[[VAL_6]]{{\[}}%[[VAL_3]] : i32] : vector<4xindex> // CHECK: %[[VAL_8:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_7]], %[[VAL_4]]], %[[VAL_5]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32> @@ -499,11 +499,11 @@ func.func @vectorize_nd_tensor_extract_block_arg(%arg0: tensor<5x6xf32>, %arg1: // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_block_arg( // CHECK-SAME: %[[VAL_0:.*]]: tensor<5x6xf32>, // CHECK-SAME: %[[VAL_1:.*]]: tensor<5xindex>) -> tensor<5xf32> { -// CHECK: %[[VAL_2:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4]> : vector<5xindex> -// CHECK: %[[VAL_4:.*]] = arith.constant dense : vector<5xi1> -// CHECK: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32> -// CHECK: %[[VAL_6:.*]] = arith.constant dense<6> : vector<5xindex> +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4]> : vector<5xindex> +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense : vector<5xi1> +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32> +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant dense<6> : vector<5xindex> // CHECK: %[[VAL_7:.*]] = tensor.empty() : tensor<5xf32> // CHECK: %[[VAL_8:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[VAL_2]] {in_bounds = [true]} : tensor<5xindex>, vector<5xindex> // CHECK: %[[VAL_9:.*]] = arith.muli %[[VAL_8]], %[[VAL_6]] : vector<5xindex> diff --git a/mlir/test/Dialect/Math/algebraic-simplification.mlir b/mlir/test/Dialect/Math/algebraic-simplification.mlir index 21c9f7a8e7f1..a97ecc52a17e 100644 --- a/mlir/test/Dialect/Math/algebraic-simplification.mlir +++ b/mlir/test/Dialect/Math/algebraic-simplification.mlir @@ -38,8 +38,8 @@ func.func @pow_cube(%arg0: f32, %arg1 : vector<4xf32>) -> (f32, vector<4xf32>) { // CHECK-LABEL: @pow_recip func.func @pow_recip(%arg0: f32, %arg1 : vector<4xf32>) -> (f32, vector<4xf32>) { - // CHECK: %[[CST_S:.*]] = arith.constant 1.0{{.*}} : f32 - // CHECK: %[[CST_V:.*]] = arith.constant dense<1.0{{.*}}> : vector<4xf32> + // CHECK-DAG: %[[CST_S:.*]] = arith.constant 1.0{{.*}} : f32 + // CHECK-DAG: %[[CST_V:.*]] = arith.constant dense<1.0{{.*}}> : vector<4xf32> // CHECK: %[[SCALAR:.*]] = arith.divf %[[CST_S]], %arg0 // CHECK: %[[VECTOR:.*]] = arith.divf %[[CST_V]], %arg1 // CHECK: return %[[SCALAR]], %[[VECTOR]] @@ -110,8 +110,8 @@ func.func @ipowi_zero_exp(%arg0: i32, %arg1: vector<4xi32>) -> (i32, vector<4xi3 // CHECK-SAME: %[[ARG1:.+]]: vector<4xi32> // CHECK-SAME: -> (i32, vector<4xi32>, i32, vector<4xi32>) { func.func @ipowi_exp_one(%arg0: i32, %arg1: vector<4xi32>) -> (i32, vector<4xi32>, i32, vector<4xi32>) { - // CHECK: %[[CST_S:.*]] = arith.constant 1 : i32 - // CHECK: %[[CST_V:.*]] = arith.constant dense<1> : vector<4xi32> + // CHECK-DAG: %[[CST_S:.*]] = arith.constant 1 : i32 + // CHECK-DAG: %[[CST_V:.*]] = arith.constant dense<1> : vector<4xi32> // CHECK: %[[SCALAR:.*]] = arith.divsi %[[CST_S]], %[[ARG0]] // CHECK: %[[VECTOR:.*]] = arith.divsi %[[CST_V]], %[[ARG1]] // CHECK: return %[[ARG0]], %[[ARG1]], %[[SCALAR]], %[[VECTOR]] @@ -131,8 +131,8 @@ func.func @ipowi_exp_one(%arg0: i32, %arg1: vector<4xi32>) -> (i32, vector<4xi32 // CHECK-SAME: %[[ARG1:.+]]: vector<4xi32> // CHECK-SAME: -> (i32, vector<4xi32>, i32, vector<4xi32>) { func.func @ipowi_exp_two(%arg0: i32, %arg1: vector<4xi32>) -> (i32, vector<4xi32>, i32, vector<4xi32>) { - // CHECK: %[[CST_S:.*]] = arith.constant 1 : i32 - // CHECK: %[[CST_V:.*]] = arith.constant dense<1> : vector<4xi32> + // CHECK-DAG: %[[CST_S:.*]] = arith.constant 1 : i32 + // CHECK-DAG: %[[CST_V:.*]] = arith.constant dense<1> : vector<4xi32> // CHECK: %[[SCALAR0:.*]] = arith.muli %[[ARG0]], %[[ARG0]] // CHECK: %[[VECTOR0:.*]] = arith.muli %[[ARG1]], %[[ARG1]] // CHECK: %[[SCALAR1:.*]] = arith.divsi %[[CST_S]], %[[ARG0]] @@ -156,8 +156,8 @@ func.func @ipowi_exp_two(%arg0: i32, %arg1: vector<4xi32>) -> (i32, vector<4xi32 // CHECK-SAME: %[[ARG1:.+]]: vector<4xi32> // CHECK-SAME: -> (i32, vector<4xi32>, i32, vector<4xi32>) { func.func @ipowi_exp_three(%arg0: i32, %arg1: vector<4xi32>) -> (i32, vector<4xi32>, i32, vector<4xi32>) { - // CHECK: %[[CST_S:.*]] = arith.constant 1 : i32 - // CHECK: %[[CST_V:.*]] = arith.constant dense<1> : vector<4xi32> + // CHECK-DAG: %[[CST_S:.*]] = arith.constant 1 : i32 + // CHECK-DAG: %[[CST_V:.*]] = arith.constant dense<1> : vector<4xi32> // CHECK: %[[SMUL0:.*]] = arith.muli %[[ARG0]], %[[ARG0]] // CHECK: %[[SCALAR0:.*]] = arith.muli %[[SMUL0]], %[[ARG0]] // CHECK: %[[VMUL0:.*]] = arith.muli %[[ARG1]], %[[ARG1]] @@ -200,8 +200,8 @@ func.func @fpowi_zero_exp(%arg0: f32, %arg1: vector<4xf32>) -> (f32, vector<4xf3 // CHECK-SAME: %[[ARG1:.+]]: vector<4xf32> // CHECK-SAME: -> (f32, vector<4xf32>, f32, vector<4xf32>) { func.func @fpowi_exp_one(%arg0: f32, %arg1: vector<4xf32>) -> (f32, vector<4xf32>, f32, vector<4xf32>) { - // CHECK: %[[CST_S:.*]] = arith.constant 1.000000e+00 : f32 - // CHECK: %[[CST_V:.*]] = arith.constant dense<1.000000e+00> : vector<4xf32> + // CHECK-DAG: %[[CST_S:.*]] = arith.constant 1.000000e+00 : f32 + // CHECK-DAG: %[[CST_V:.*]] = arith.constant dense<1.000000e+00> : vector<4xf32> // CHECK: %[[SCALAR:.*]] = arith.divf %[[CST_S]], %[[ARG0]] // CHECK: %[[VECTOR:.*]] = arith.divf %[[CST_V]], %[[ARG1]] // CHECK: return %[[ARG0]], %[[ARG1]], %[[SCALAR]], %[[VECTOR]] @@ -221,8 +221,8 @@ func.func @fpowi_exp_one(%arg0: f32, %arg1: vector<4xf32>) -> (f32, vector<4xf32 // CHECK-SAME: %[[ARG1:.+]]: vector<4xf32> // CHECK-SAME: -> (f32, vector<4xf32>, f32, vector<4xf32>) { func.func @fpowi_exp_two(%arg0: f32, %arg1: vector<4xf32>) -> (f32, vector<4xf32>, f32, vector<4xf32>) { - // CHECK: %[[CST_S:.*]] = arith.constant 1.000000e+00 : f32 - // CHECK: %[[CST_V:.*]] = arith.constant dense<1.000000e+00> : vector<4xf32> + // CHECK-DAG: %[[CST_S:.*]] = arith.constant 1.000000e+00 : f32 + // CHECK-DAG: %[[CST_V:.*]] = arith.constant dense<1.000000e+00> : vector<4xf32> // CHECK: %[[SCALAR0:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] // CHECK: %[[VECTOR0:.*]] = arith.mulf %[[ARG1]], %[[ARG1]] // CHECK: %[[SCALAR1:.*]] = arith.divf %[[CST_S]], %[[ARG0]] @@ -246,8 +246,8 @@ func.func @fpowi_exp_two(%arg0: f32, %arg1: vector<4xf32>) -> (f32, vector<4xf32 // CHECK-SAME: %[[ARG1:.+]]: vector<4xf32> // CHECK-SAME: -> (f32, vector<4xf32>, f32, vector<4xf32>) { func.func @fpowi_exp_three(%arg0: f32, %arg1: vector<4xf32>) -> (f32, vector<4xf32>, f32, vector<4xf32>) { - // CHECK: %[[CST_S:.*]] = arith.constant 1.000000e+00 : f32 - // CHECK: %[[CST_V:.*]] = arith.constant dense<1.000000e+00> : vector<4xf32> + // CHECK-DAG: %[[CST_S:.*]] = arith.constant 1.000000e+00 : f32 + // CHECK-DAG: %[[CST_V:.*]] = arith.constant dense<1.000000e+00> : vector<4xf32> // CHECK: %[[SMUL0:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] // CHECK: %[[SCALAR0:.*]] = arith.mulf %[[SMUL0]], %[[ARG0]] // CHECK: %[[VMUL0:.*]] = arith.mulf %[[ARG1]], %[[ARG1]] diff --git a/mlir/test/Dialect/Math/expand-math.mlir b/mlir/test/Dialect/Math/expand-math.mlir index 3e1c3462fc8f..6ee65b085dad 100644 --- a/mlir/test/Dialect/Math/expand-math.mlir +++ b/mlir/test/Dialect/Math/expand-math.mlir @@ -65,18 +65,18 @@ func.func @ctlz(%arg: i32) -> i32 { // CHECK-LABEL: @ctlz // CHECK-SAME: %[[ARG0:.+]]: i32 -// CHECK-DAG: %[[C0:.+]] = arith.constant 0 -// CHECK-DAG: %[[C16:.+]] = arith.constant 16 -// CHECK-DAG: %[[C65535:.+]] = arith.constant 65535 -// CHECK-DAG: %[[C8:.+]] = arith.constant 8 -// CHECK-DAG: %[[C16777215:.+]] = arith.constant 16777215 -// CHECK-DAG: %[[C4:.+]] = arith.constant 4 -// CHECK-DAG: %[[C268435455:.+]] = arith.constant 268435455 -// CHECK-DAG: %[[C2:.+]] = arith.constant 2 -// CHECK-DAG: %[[C1073741823:.+]] = arith.constant 1073741823 -// CHECK-DAG: %[[C1:.+]] = arith.constant 1 -// CHECK-DAG: %[[C2147483647:.+]] = arith.constant 2147483647 -// CHECK-DAG: %[[C32:.+]] = arith.constant 32 +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : i32 +// CHECK-DAG: %[[C16:.+]] = arith.constant 16 : i32 +// CHECK-DAG: %[[C65535:.+]] = arith.constant 65535 : i32 +// CHECK-DAG: %[[C8:.+]] = arith.constant 8 : i32 +// CHECK-DAG: %[[C16777215:.+]] = arith.constant 16777215 : i32 +// CHECK-DAG: %[[C4:.+]] = arith.constant 4 : i32 +// CHECK-DAG: %[[C268435455:.+]] = arith.constant 268435455 : i32 +// CHECK-DAG: %[[C2:.+]] = arith.constant 2 : i32 +// CHECK-DAG: %[[C1073741823:.+]] = arith.constant 1073741823 : i32 +// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : i32 +// CHECK-DAG: %[[C2147483647:.+]] = arith.constant 2147483647 : i32 +// CHECK-DAG: %[[C32:.+]] = arith.constant 32 : i32 // CHECK: %[[PRED:.+]] = arith.cmpi ule, %[[ARG0]], %[[C65535]] // CHECK: %[[SHL:.+]] = arith.shli %[[ARG0]], %[[C16]] diff --git a/mlir/test/Dialect/Math/polynomial-approximation.mlir b/mlir/test/Dialect/Math/polynomial-approximation.mlir index 6743998d6162..834a7dc0af66 100644 --- a/mlir/test/Dialect/Math/polynomial-approximation.mlir +++ b/mlir/test/Dialect/Math/polynomial-approximation.mlir @@ -279,28 +279,28 @@ func.func @expm1_vector(%arg0: vector<8x8xf32>) -> vector<8x8xf32> { // CHECK-LABEL: func @log_scalar( // CHECK-SAME: %[[X:.*]]: f32) -> f32 { -// CHECK: %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[VAL_2:.*]] = arith.constant 1.000000e+00 : f32 -// CHECK: %[[VAL_3:.*]] = arith.constant -5.000000e-01 : f32 -// CHECK: %[[VAL_4:.*]] = arith.constant 1.17549435E-38 : f32 -// CHECK: %[[VAL_5:.*]] = arith.constant 0xFF800000 : f32 -// CHECK: %[[VAL_6:.*]] = arith.constant 0x7F800000 : f32 -// CHECK: %[[VAL_7:.*]] = arith.constant 0x7FC00000 : f32 -// CHECK: %[[VAL_8:.*]] = arith.constant 0.707106769 : f32 -// CHECK: %[[VAL_9:.*]] = arith.constant 0.0703768358 : f32 -// CHECK: %[[VAL_10:.*]] = arith.constant -0.115146101 : f32 -// CHECK: %[[VAL_11:.*]] = arith.constant 0.116769984 : f32 -// CHECK: %[[VAL_12:.*]] = arith.constant -0.12420141 : f32 -// CHECK: %[[VAL_13:.*]] = arith.constant 0.142493233 : f32 -// CHECK: %[[VAL_14:.*]] = arith.constant -0.166680574 : f32 -// CHECK: %[[VAL_15:.*]] = arith.constant 0.200007141 : f32 -// CHECK: %[[VAL_16:.*]] = arith.constant -0.24999994 : f32 -// CHECK: %[[VAL_17:.*]] = arith.constant 0.333333313 : f32 -// CHECK: %[[VAL_18:.*]] = arith.constant 1.260000e+02 : f32 -// CHECK: %[[VAL_19:.*]] = arith.constant -2139095041 : i32 -// CHECK: %[[VAL_20:.*]] = arith.constant 1056964608 : i32 -// CHECK: %[[VAL_21:.*]] = arith.constant 23 : i32 -// CHECK: %[[VAL_22:.*]] = arith.constant 0.693147182 : f32 +// CHECK-DAG: %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 1.000000e+00 : f32 +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant -5.000000e-01 : f32 +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 1.17549435E-38 : f32 +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 0xFF800000 : f32 +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0x7F800000 : f32 +// CHECK-DAG: %[[VAL_7:.*]] = arith.constant 0x7FC00000 : f32 +// CHECK-DAG: %[[VAL_8:.*]] = arith.constant 0.707106769 : f32 +// CHECK-DAG: %[[VAL_9:.*]] = arith.constant 0.0703768358 : f32 +// CHECK-DAG: %[[VAL_10:.*]] = arith.constant -0.115146101 : f32 +// CHECK-DAG: %[[VAL_11:.*]] = arith.constant 0.116769984 : f32 +// CHECK-DAG: %[[VAL_12:.*]] = arith.constant -0.12420141 : f32 +// CHECK-DAG: %[[VAL_13:.*]] = arith.constant 0.142493233 : f32 +// CHECK-DAG: %[[VAL_14:.*]] = arith.constant -0.166680574 : f32 +// CHECK-DAG: %[[VAL_15:.*]] = arith.constant 0.200007141 : f32 +// CHECK-DAG: %[[VAL_16:.*]] = arith.constant -0.24999994 : f32 +// CHECK-DAG: %[[VAL_17:.*]] = arith.constant 0.333333313 : f32 +// CHECK-DAG: %[[VAL_18:.*]] = arith.constant 1.260000e+02 : f32 +// CHECK-DAG: %[[VAL_19:.*]] = arith.constant -2139095041 : i32 +// CHECK-DAG: %[[VAL_20:.*]] = arith.constant 1056964608 : i32 +// CHECK-DAG: %[[VAL_21:.*]] = arith.constant 23 : i32 +// CHECK-DAG: %[[VAL_22:.*]] = arith.constant 0.693147182 : f32 // CHECK: %[[VAL_23:.*]] = arith.cmpf ugt, %[[X]], %[[VAL_4]] : f32 // CHECK: %[[VAL_24:.*]] = arith.select %[[VAL_23]], %[[X]], %[[VAL_4]] : f32 // CHECK-NOT: frexp @@ -417,20 +417,20 @@ func.func @log1p_vector(%arg0: vector<8xf32>) -> vector<8xf32> { // CHECK-LABEL: func @tanh_scalar( // CHECK-SAME: %[[VAL_0:.*]]: f32) -> f32 { -// CHECK: %[[VAL_1:.*]] = arith.constant -7.99881172 : f32 -// CHECK: %[[VAL_2:.*]] = arith.constant 7.99881172 : f32 -// CHECK: %[[VAL_3:.*]] = arith.constant 4.000000e-04 : f32 -// CHECK: %[[VAL_4:.*]] = arith.constant 0.00489352457 : f32 -// CHECK: %[[VAL_5:.*]] = arith.constant 6.37261954E-4 : f32 -// CHECK: %[[VAL_6:.*]] = arith.constant 1.48572235E-5 : f32 -// CHECK: %[[VAL_7:.*]] = arith.constant 5.12229725E-8 : f32 -// CHECK: %[[VAL_8:.*]] = arith.constant -8.60467184E-11 : f32 -// CHECK: %[[VAL_9:.*]] = arith.constant 2.00018794E-13 : f32 -// CHECK: %[[VAL_10:.*]] = arith.constant -2.76076837E-16 : f32 -// CHECK: %[[VAL_11:.*]] = arith.constant 0.00489352504 : f32 -// CHECK: %[[VAL_12:.*]] = arith.constant 0.00226843474 : f32 -// CHECK: %[[VAL_13:.*]] = arith.constant 1.18534706E-4 : f32 -// CHECK: %[[VAL_14:.*]] = arith.constant 1.19825836E-6 : f32 +// CHECK-DAG: %[[VAL_1:.*]] = arith.constant -7.99881172 : f32 +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 7.99881172 : f32 +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 4.000000e-04 : f32 +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 0.00489352457 : f32 +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 6.37261954E-4 : f32 +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 1.48572235E-5 : f32 +// CHECK-DAG: %[[VAL_7:.*]] = arith.constant 5.12229725E-8 : f32 +// CHECK-DAG: %[[VAL_8:.*]] = arith.constant -8.60467184E-11 : f32 +// CHECK-DAG: %[[VAL_9:.*]] = arith.constant 2.00018794E-13 : f32 +// CHECK-DAG: %[[VAL_10:.*]] = arith.constant -2.76076837E-16 : f32 +// CHECK-DAG: %[[VAL_11:.*]] = arith.constant 0.00489352504 : f32 +// CHECK-DAG: %[[VAL_12:.*]] = arith.constant 0.00226843474 : f32 +// CHECK-DAG: %[[VAL_13:.*]] = arith.constant 1.18534706E-4 : f32 +// CHECK-DAG: %[[VAL_14:.*]] = arith.constant 1.19825836E-6 : f32 // CHECK: %[[VAL_15:.*]] = arith.cmpf ult, %[[VAL_0]], %[[VAL_2]] : f32 // CHECK: %[[VAL_16:.*]] = arith.select %[[VAL_15]], %[[VAL_0]], %[[VAL_2]] : f32 // CHECK: %[[VAL_17:.*]] = arith.cmpf ugt, %[[VAL_16]], %[[VAL_1]] : f32 diff --git a/mlir/test/Dialect/Mesh/resharding-spmdization.mlir b/mlir/test/Dialect/Mesh/resharding-spmdization.mlir index 0ba0d76c09a7..c7088fe646d8 100644 --- a/mlir/test/Dialect/Mesh/resharding-spmdization.mlir +++ b/mlir/test/Dialect/Mesh/resharding-spmdization.mlir @@ -19,8 +19,8 @@ func.func @split_replicated_tensor_axis( // CHECK-SAME: %[[ARG:.*]]: tensor<3x14xf32> %arg0: tensor<3x14xf32> ) -> tensor<3x14xf32> { - // CHECK: %[[ZERO:.*]] = arith.constant 0 : index - // CHECK: %[[TENSOR_SPLIT_AXIS_SIZE:.*]] = arith.constant 14 : index + // CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : index + // CHECK-DAG: %[[TENSOR_SPLIT_AXIS_SIZE:.*]] = arith.constant 14 : index // CHECK: %[[PROCESS_INDEX:.*]] = mesh.process_index on @mesh_1d axes = [0] : index // CHECK: %[[MESH_AXIS_SIZE:.*]] = mesh.cluster_shape @mesh_1d axes = [0] : index // CHECK: %[[TENSOR_SPLIT_AXIS_SIZE_MOD_MESH_AXIS_SIZE:.*]] = arith.remui %[[TENSOR_SPLIT_AXIS_SIZE]], %[[MESH_AXIS_SIZE]] : index @@ -41,8 +41,8 @@ func.func @split_replicated_tensor_axis_dynamic( // CHECK-SAME: %[[ARG:.*]]: tensor %arg0: tensor ) -> tensor { - // CHECK: %[[ZERO:.*]] = arith.constant 0 : index - // CHECK: %[[TWO:.*]] = arith.constant 2 : index + // CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : index + // CHECK-DAG: %[[TWO:.*]] = arith.constant 2 : index // CHECK: %[[PROCESS_INDEX:.*]] = mesh.process_index on @mesh_1d_dynamic axes = [0] : index // CHECK: %[[MESH_AXIS_SIZE:.*]] = mesh.cluster_shape @mesh_1d_dynamic axes = [0] : index // CHECK: %[[TENSOR_SPLIT_AXIS_SIZE:.*]] = tensor.dim %[[ARG]], %[[ZERO]] : tensor diff --git a/mlir/test/Dialect/NVGPU/transform-create-async-groups.mlir b/mlir/test/Dialect/NVGPU/transform-create-async-groups.mlir index e1e4111adca8..8290001c4585 100644 --- a/mlir/test/Dialect/NVGPU/transform-create-async-groups.mlir +++ b/mlir/test/Dialect/NVGPU/transform-create-async-groups.mlir @@ -169,9 +169,9 @@ builtin.module { // CHECK-LABEL: @read_2d_with_mask( // CHECK-SAME: %[[sz0:.*]]: index, %[[sz1:.*]]: index, %[[a:.*]]: memref<1024x1024xf32> func.func @read_2d_with_mask(%sz0: index, %sz1: index, %a: memref<1024x1024xf32>) { - // CHECK: %[[c0:.*]] = arith.constant 0 : index - // CHECK: %[[c1:.*]] = arith.constant 1 : index - // CHECK: %[[c2:.*]] = arith.constant 2 : index + // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index + // CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index + // CHECK-DAG: %[[c2:.*]] = arith.constant 2 : index %0 = memref.alloc() : memref<4x32x16xf32, #gpu.address_space> %c0 = arith.constant 0 : index %cst_0 = arith.constant 0.000000e+00 : f32 @@ -215,9 +215,9 @@ builtin.module { // CHECK-LABEL: @read_3d_with_mask( // CHECK-SAME: %[[sz0:.*]]: index, %[[sz1:.*]]: index, %[[sz2:.*]]: index, %[[a:.*]]: memref<1024x1024x1024xf32> func.func @read_3d_with_mask(%sz0: index, %sz1: index, %sz2: index, %a: memref<1024x1024x1024xf32>) { - // CHECK: %[[c0:.*]] = arith.constant 0 : index - // CHECK: %[[c1:.*]] = arith.constant 1 : index - // CHECK: %[[c2:.*]] = arith.constant 2 : index + // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index + // CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index + // CHECK-DAG: %[[c2:.*]] = arith.constant 2 : index %0 = memref.alloc() : memref<4x32x16xf32, #gpu.address_space> %c0 = arith.constant 0 : index %cst_0 = arith.constant 0.000000e+00 : f32 diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir index a18c850c3f05..33290d2db31d 100644 --- a/mlir/test/Dialect/SCF/loop-pipelining.mlir +++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir @@ -547,9 +547,9 @@ func.func @backedge_same_stage(%A: memref) -> f32 { // ----- // CHECK: @pipeline_op_with_region(%[[ARG0:.+]]: memref, %[[ARG1:.+]]: memref, %[[ARG2:.+]]: memref, %[[CF:.*]]: f32) { -// CHECK: %[[C0:.+]] = arith.constant 0 : -// CHECK: %[[C3:.+]] = arith.constant 3 : -// CHECK: %[[C1:.+]] = arith.constant 1 : +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : +// CHECK-DAG: %[[C3:.+]] = arith.constant 3 : +// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : // CHECK: %[[APRO:.+]] = memref.alloc() : // CHECK: %[[BPRO:.+]] = memref.alloc() : // CHECK: %[[ASV0:.+]] = memref.subview %[[ARG0]][%[[C0]]] [8] [1] : diff --git a/mlir/test/Dialect/SparseTensor/sparse_1d.mlir b/mlir/test/Dialect/SparseTensor/sparse_1d.mlir index 8f96f3379487..fcc221660353 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_1d.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_1d.mlir @@ -45,10 +45,10 @@ func.func @add_d(%arga: tensor<32xf32, #DV>, %argb: f32, %argx: tensor<32xf32>) // CHECK-LABEL: func @add_d_init( // CHECK-SAME: %[[VAL_0:.*]]: tensor<32xf32, #sparse{{[0-9]*}}>, // CHECK-SAME: %[[VAL_1:.*]]: f32) -> tensor<32xf32> { -// CHECK: %[[VAL_2:.*]] = arith.constant 32 : index -// CHECK: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32 -// CHECK: %[[VAL_4:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_5:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 32 : index +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 1 : index // CHECK: %[[VAL_INITTENSOR:.*]] = tensor.empty() : tensor<32xf32> // CHECK: %[[VAL_6:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xf32, #sparse{{[0-9]*}}> to memref // CHECK: %[[VAL_7:.*]] = bufferization.to_memref %[[VAL_INITTENSOR]] : memref<32xf32> diff --git a/mlir/test/Dialect/SparseTensor/sparse_affine.mlir b/mlir/test/Dialect/SparseTensor/sparse_affine.mlir index aa75e20460f5..886b21fa9756 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_affine.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_affine.mlir @@ -58,10 +58,10 @@ func.func @mul_inv_dense1d(%arga: tensor<32xf32, #SpVec>, // CHECK-LABEL: func.func @mul_inv_enc_dense1d( // CHECK-SAME: %[[VAL_0:.*]]: tensor<32xf32, #sparse{{[0-9]*}}>, // CHECK-SAME: %[[VAL_1:.*]]: tensor<4xf32, #sparse{{[0-9]*}}>) -> tensor<32xf32, #sparse{{[0-9]*}}> { -// CHECK: %[[VAL_2:.*]] = arith.constant 32 : index -// CHECK: %[[VAL_3:.*]] = arith.constant 3 : index -// CHECK: %[[VAL_4:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_5:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 32 : index +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 1 : index // CHECK: %[[VAL_6:.*]] = tensor.empty() : tensor<32xf32, #sparse{{[0-9]*}}> // CHECK: %[[VAL_7:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xf32, #sparse{{[0-9]*}}> to memref // CHECK: %[[VAL_8:.*]] = sparse_tensor.values %[[VAL_1]] : tensor<4xf32, #sparse{{[0-9]*}}> to memref diff --git a/mlir/test/Dialect/SparseTensor/sparse_concat.mlir b/mlir/test/Dialect/SparseTensor/sparse_concat.mlir index a323fa85d2fd..bbf0b7c7c341 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_concat.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_concat.mlir @@ -10,10 +10,10 @@ // CHECK-SAME: %[[TMP_arg0:.*]]: tensor<2x4xf64, #sparse> // CHECK-SAME: %[[TMP_arg1:.*]]: tensor<3x4xf64, #sparse> // CHECK-SAME: %[[TMP_arg2:.*]]: tensor<4x4xf64, #sparse> -// CHECK: %[[TMP_c0:.*]] = arith.constant 0 : index -// CHECK: %[[TMP_c1:.*]] = arith.constant 1 : index -// CHECK: %[[TMP_c5:.*]] = arith.constant 5 : index -// CHECK: %[[TMP_c2:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[TMP_c0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[TMP_c1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[TMP_c5:.*]] = arith.constant 5 : index +// CHECK-DAG: %[[TMP_c2:.*]] = arith.constant 2 : index // CHECK: %[[TMP_0:.*]] = bufferization.alloc_tensor() : tensor<9x4xf64, #sparse> // CHECK: %[[TMP_1:.*]] = sparse_tensor.positions %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse> // CHECK: %[[TMP_2:.*]] = sparse_tensor.coordinates %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse> diff --git a/mlir/test/Dialect/SparseTensor/sparse_storage.mlir b/mlir/test/Dialect/SparseTensor/sparse_storage.mlir index a8cc056139e4..cd55ba31a6bc 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_storage.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_storage.mlir @@ -23,8 +23,8 @@ } // CHECK-LABEL: func @mul64( -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK: %[[P0:.*]] = memref.load %{{.*}}[%[[C0]]] : memref // CHECK: %[[B0:.*]] = arith.index_cast %[[P0]] : i64 to index // CHECK: %[[P1:.*]] = memref.load %{{.*}}[%[[C1]]] : memref @@ -49,8 +49,8 @@ func.func @mul64(%arga: tensor<32xf64, #SparseVector64>, %argb: tensor<32xf64>, } // CHECK-LABEL: func @mul32( -// CHECK: %[[C0:.*]] = arith.constant 0 : index -// CHECK: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index // CHECK: %[[P0:.*]] = memref.load %{{.*}}[%[[C0]]] : memref // CHECK: %[[Z0:.*]] = arith.extui %[[P0]] : i32 to i64 // CHECK: %[[B0:.*]] = arith.index_cast %[[Z0]] : i64 to index diff --git a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir index ca4eb4ff6794..e9e3ca9c0087 100644 --- a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir +++ b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir @@ -261,8 +261,8 @@ func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_dims(%arg0: tensor<56 } // CHECK: func @tensor_pack_linalg_transpose_fold_dynamic_outer_dims( // CHECK-SAME: %[[ARG0:.+]]: tensor<56x?x?x64xf32>) -// CHECK: %[[c1:.+]] = arith.constant 1 : index -// CHECK: %[[c2:.+]] = arith.constant 2 : index +// CHECK-DAG: %[[c1:.+]] = arith.constant 1 : index +// CHECK-DAG: %[[c2:.+]] = arith.constant 2 : index // CHECK: %[[dim:.+]] = tensor.dim %[[ARG0]], %[[c1]] : tensor<56x?x?x64xf32> // CHECK: %[[dim_0:.+]] = tensor.dim %[[ARG0]], %[[c2]] : tensor<56x?x?x64xf32> // CHECK: %[[INIT:.+]] = tensor.empty(%[[dim_0]], %[[dim]]) : tensor @@ -295,8 +295,8 @@ func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_and_tile_dims(%arg0: // CHECK: module { // CHECK: func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_and_tile_dims( // CHECK-SAME: %[[ARG0:.+]]: tensor<56x?x?x128xf32>) -// CHECK: %[[c1:.+]] = arith.constant 1 : index -// CHECK: %[[c2:.+]] = arith.constant 2 : index +// CHECK-DAG: %[[c1:.+]] = arith.constant 1 : index +// CHECK-DAG: %[[c2:.+]] = arith.constant 2 : index // CHECK: %[[dim:.+]] = tensor.dim %[[ARG0]], %[[c1]] : tensor<56x?x?x128xf32> // CHECK: %[[dim_0:.+]] = tensor.dim %[[ARG0]], %[[c2]] : tensor<56x?x?x128xf32> // CHECK: %[[mapped_dim1:.+]] = affine.apply #[[map:.+]]()[%[[dim]]] @@ -330,10 +330,10 @@ func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_dims_tile_dims_tile_s // CHECK-SAME: %[[PACK_DEST:.+]]: tensor, %[[TRANSPOSE_DEST:.+]]: tensor, // CHECK-SAME: %[[ARG1:.+]]: index, %[[ARG2:.+]]: index, // CHECK-SAME: %[[ARG3:.+]]: index) -// CHECK: %[[c0:.+]] = arith.constant 0 : index -// CHECK: %[[c1:.+]] = arith.constant 1 : index -// CHECK: %[[c2:.+]] = arith.constant 2 : index -// CHECK: %[[c3:.+]] = arith.constant 3 : index +// CHECK-DAG: %[[c0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[c1:.+]] = arith.constant 1 : index +// CHECK-DAG: %[[c2:.+]] = arith.constant 2 : index +// CHECK-DAG: %[[c3:.+]] = arith.constant 3 : index // CHECK: %[[dim:.+]] = tensor.dim %[[ARG0]], %[[c0]] : tensor // CHECK: %[[dim_0:.+]] = tensor.dim %[[ARG0]], %[[c1]] : tensor // CHECK: %[[dim_1:.+]] = tensor.dim %[[ARG0]], %[[c2]] : tensor diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir index 612e99f19851..27ca3ae3c21b 100644 --- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir +++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir @@ -1080,9 +1080,9 @@ func.func @reduce_sum_constant_aggressive() -> tensor<1x3xi32> { func.func @reduce_sum_constant_aggressive() -> tensor<2x3xi32> { // AGGRESIVE-LABEL: func.func @reduce_sum_constant_aggressive() -> tensor<2x3xi32> { - // AGGRESIVE: %[[VAL_0:.*]] = "tosa.const"() <{value = dense<2> : tensor<1x2x3xi32>}> : () -> tensor<1x2x3xi32> - // AGGRESIVE: %[[VAL_1:.*]] = "tosa.const"() <{value = dense<1> : tensor<2x2x3xi32>}> : () -> tensor<2x2x3xi32> - // AGGRESIVE: %[[VAL_2:.*]] = "tosa.const"() <{value = dense<2> : tensor<2x3xi32>}> : () -> tensor<2x3xi32> + // AGGRESIVE-DAG: %[[VAL_0:.*]] = "tosa.const"() <{value = dense<2> : tensor<1x2x3xi32>}> : () -> tensor<1x2x3xi32> + // AGGRESIVE-DAG: %[[VAL_1:.*]] = "tosa.const"() <{value = dense<1> : tensor<2x2x3xi32>}> : () -> tensor<2x2x3xi32> + // AGGRESIVE-DAG: %[[VAL_2:.*]] = "tosa.const"() <{value = dense<2> : tensor<2x3xi32>}> : () -> tensor<2x3xi32> // AGGRESIVE: %[[VAL_3:.*]] = tosa.argmax %[[VAL_0]] {axis = 0 : i32} : (tensor<1x2x3xi32>) -> tensor<2x3xi32> // AGGRESIVE: %[[VAL_4:.*]] = tosa.argmax %[[VAL_1]] {axis = 0 : i32} : (tensor<2x2x3xi32>) -> tensor<2x3xi32> // AGGRESIVE: %[[VAL_5:.*]] = tosa.add %[[VAL_3]], %[[VAL_2]] : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32> diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir index c86bf5d056f8..2224bf3f57b2 100644 --- a/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir +++ b/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir @@ -26,8 +26,8 @@ func.func @depthwise_conv2d_as_mul(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1 // CHECK-LABEL: @depthwise_conv2d_as_mul_q func.func @depthwise_conv2d_as_mul_q(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<1x1x2x3xi8>, %arg2: tensor<6xi32>) -> tensor<4x10x10x6xi32> { - // CHECK: %[[iZp:.+]] = "tosa.const"() <{value = dense<7> : tensor<1x1x1x1x1xi32>} - // CHECK: %[[wZp:.+]] = "tosa.const"() <{value = dense<11> : tensor<1x1x1x1xi32>} + // CHECK-DAG: %[[iZp:.+]] = "tosa.const"() <{value = dense<7> : tensor<1x1x1x1x1xi32>} + // CHECK-DAG: %[[wZp:.+]] = "tosa.const"() <{value = dense<11> : tensor<1x1x1x1xi32>} // CHECK: %[[rIn:.+]] = tosa.reshape %arg0 {new_shape = array} // CHECK: %[[cIn:.+]] = tosa.cast %[[rIn]] : (tensor<4x10x10x2x1xi8>) -> tensor<4x10x10x2x1xi32> // CHECK: %[[cWe:.+]] = tosa.cast %arg1 : (tensor<1x1x2x3xi8>) -> tensor<1x1x2x3xi32> @@ -46,8 +46,8 @@ func.func @depthwise_conv2d_as_mul_q(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor< // CHECK-LABEL: @depthwise_conv2d_as_mul_padded func.func @depthwise_conv2d_as_mul_padded(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1x1x2x3xf32>, %arg2: tensor<6xf32>) -> tensor<4x12x12x6xf32> { - // CHECK: %[[pad:.+]] = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0], [0, 0]]> : tensor<5x2xi64>} - // CHECK: %[[zero:.+]] = "tosa.const"() <{value = dense<0.000000e+00> : tensor} + // CHECK-DAG: %[[pad:.+]] = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0], [0, 0]]> : tensor<5x2xi64>} + // CHECK-DAG: %[[zero:.+]] = "tosa.const"() <{value = dense<0.000000e+00> : tensor} // CHECK: %[[reIn:.+]] = tosa.reshape %arg0 {new_shape = array} // CHECK: %[[padded:.+]] = tosa.pad %[[reIn]], %[[pad]], %[[zero]] : (tensor<4x10x10x2x1xf32>, tensor<5x2xi64>, tensor) -> tensor<4x12x12x2x1xf32> // CHECK: %[[reArg1:.+]] = tosa.reshape %arg1 {new_shape = array} diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir index 5f9fe68c5789..1f2bb3fb9a36 100644 --- a/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir +++ b/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir @@ -108,15 +108,12 @@ func.func @transpose_conv2d_strided_quantized(%arg0: tensor<2x17x15x3xi8>, %arg1 // CHECK-LABEL: @transpose_conv2d_strided_overpad func.func @transpose_conv2d_strided_overpad(%arg0 : tensor<1x16x1x1xi8>, %arg1 : tensor<1x2x1x1xi8>, %arg2 : tensor<1xi32>) -> (tensor<1x19x2x1xi32>) { - // CHECK: %[[WEIGHT_PAD:.+]] = "tosa.const"() - // CHECK-SAME{literal}: value = dense<[[0, 0], [0, 0], [0, 1], [0, 0]]> : tensor<4x2xi32> - // CHECK: %[[WEIGHT_PERMS:.+]] = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>} - // CHECK: %[[INPUT_PAD:.+]] = "tosa.const"() - // CHECK-SAME{literal}: value = dense<[[0, 0], [1, 1], [0, 0], [0, 0]]> : tensor<4x2xi32>} - // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0> : tensor<2xi32>} - // CHECK: %[[RESULT_PERMS:.+]] = "tosa.const"() <{value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>} - // CHECK: %[[RESULT_PAD:.+]] = "tosa.const"() - // CHECK-SAME{literal}: value = dense<[[0, 0], [2, 0], [0, 0], [0, 0]]> : tensor<4x2xi32>} + // CHECK-DAG: %[[WEIGHT_PAD:.+]] = "tosa.const"() <{value = dense<{{\[}}[0, 0], [0, 0], [0, 1], [0, 0]]> : tensor<4x2xi32> + // CHECK-DAG: %[[WEIGHT_PERMS:.+]] = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>} + // CHECK-DAG: %[[INPUT_PAD:.+]] = "tosa.const"() <{value = dense<{{\[}}[0, 0], [1, 1], [0, 0], [0, 0]]> : tensor<4x2xi32>} + // CHECK-DAG: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0> : tensor<2xi32>} + // CHECK-DAG: %[[RESULT_PERMS:.+]] = "tosa.const"() <{value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>} + // CHECK-DAG: %[[RESULT_PAD:.+]] = "tosa.const"() <{value = dense<{{\[}}[0, 0], [2, 0], [0, 0], [0, 0]]> : tensor<4x2xi32>} // CHECK: %[[PAD_WEIGHT:.+]] = tosa.pad %arg1, %[[WEIGHT_PAD]] {quantization_info = #tosa.pad_quant} // CHECK: %[[RESHAPE_WEIGHT_0:.+]] = tosa.reshape %[[PAD_WEIGHT]] {new_shape = array} // CHECK: %[[TRANSPOSE_WEIGHT:.+]] = tosa.transpose %[[RESHAPE_WEIGHT_0]], %[[WEIGHT_PERMS]] diff --git a/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir b/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir index 8774a7513c9a..4a5ea439134c 100644 --- a/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-broadcast-lowering-transforms.mlir @@ -54,8 +54,8 @@ func.func @broadcast_vec2d_from_vec1d(%arg0: vector<2xf32>) -> vector<3x2xf32> { // CHECK-LABEL: func @broadcast_vec3d_from_vec1d // CHECK-SAME: %[[A:.*0]]: vector<2xf32> -// CHECK: %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32> -// CHECK: %[[C1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32> +// CHECK-DAG: %[[C0:.*]] = arith.constant dense<0.000000e+00> : vector<3x2xf32> +// CHECK-DAG: %[[C1:.*]] = arith.constant dense<0.000000e+00> : vector<4x3x2xf32> // CHECK: %[[T0:.*]] = vector.insert %[[A]], %[[C0]] [0] : vector<2xf32> into vector<3x2xf32> // CHECK: %[[T1:.*]] = vector.insert %[[A]], %[[T0]] [1] : vector<2xf32> into vector<3x2xf32> // CHECK: %[[T2:.*]] = vector.insert %[[A]], %[[T1]] [2] : vector<2xf32> into vector<3x2xf32> diff --git a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir index 3c5539d1aea6..78cf82e1ab6c 100644 --- a/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-contract-to-matrix-intrinsics-transforms.mlir @@ -14,9 +14,9 @@ // CHECK-SAME: %[[A:[a-zA-Z0-9]*]]: vector<2x4xf32>, // CHECK-SAME: %[[B:[a-zA-Z0-9]*]]: vector<4x3xf32>, // CHECK-SAME: %[[C:[a-zA-Z0-9]*]]: vector<2x3xf32> -// CHECK: %[[vcst:.*]] = arith.constant dense<0.000000e+00> : vector<8xf32> -// CHECK: %[[vcst_0:.*]] = arith.constant dense<0.000000e+00> : vector<12xf32> -// CHECK: %[[vcst_1:.*]] = arith.constant dense<0.000000e+00> : vector<2x3xf32> +// CHECK-DAG: %[[vcst:.*]] = arith.constant dense<0.000000e+00> : vector<8xf32> +// CHECK-DAG: %[[vcst_0:.*]] = arith.constant dense<0.000000e+00> : vector<12xf32> +// CHECK-DAG: %[[vcst_1:.*]] = arith.constant dense<0.000000e+00> : vector<2x3xf32> // CHECK: %[[a0:.*]] = vector.extract %[[A]][0] : vector<4xf32> from vector<2x4xf32> // CHECK: %[[a1:.*]] = vector.insert_strided_slice %[[a0]], %[[vcst]] {offsets = [0], strides = [1]} : vector<4xf32> into vector<8xf32> // CHECK: %[[a2:.*]] = vector.extract %[[A]][1] : vector<4xf32> from vector<2x4xf32> diff --git a/mlir/test/Dialect/Vector/vector-mask-lowering-transforms.mlir b/mlir/test/Dialect/Vector/vector-mask-lowering-transforms.mlir index f74af8eda0e6..7838543e151b 100644 --- a/mlir/test/Dialect/Vector/vector-mask-lowering-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-mask-lowering-transforms.mlir @@ -22,9 +22,9 @@ func.func @genbool_2d() -> vector<4x4xi1> { } // CHECK-LABEL: func @genbool_3d -// CHECK: %[[C1:.*]] = arith.constant dense<[true, true, true, false]> : vector<4xi1> -// CHECK: %[[C2:.*]] = arith.constant dense : vector<3x4xi1> -// CHECK: %[[C3:.*]] = arith.constant dense : vector<2x3x4xi1> +// CHECK-DAG: %[[C1:.*]] = arith.constant dense<[true, true, true, false]> : vector<4xi1> +// CHECK-DAG: %[[C2:.*]] = arith.constant dense : vector<3x4xi1> +// CHECK-DAG: %[[C3:.*]] = arith.constant dense : vector<2x3x4xi1> // CHECK: %[[T0:.*]] = vector.insert %[[C1]], %[[C2]] [0] : vector<4xi1> into vector<3x4xi1> // CHECK: %[[T1:.*]] = vector.insert %[[T0]], %[[C3]] [0] : vector<3x4xi1> into vector<2x3x4xi1> // CHECK: return %[[T1]] : vector<2x3x4xi1> @@ -47,10 +47,10 @@ func.func @genbool_var_1d(%arg0: index) -> vector<3xi1> { // CHECK-LABEL: func @genbool_var_2d( // CHECK-SAME: %[[A:.*0]]: index, // CHECK-SAME: %[[B:.*1]]: index) -// CHECK: %[[C1:.*]] = arith.constant dense : vector<3xi1> -// CHECK: %[[C2:.*]] = arith.constant dense : vector<2x3xi1> -// CHECK: %[[c0:.*]] = arith.constant 0 : index -// CHECK: %[[c1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant dense : vector<3xi1> +// CHECK-DAG: %[[C2:.*]] = arith.constant dense : vector<2x3xi1> +// CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index // CHECK: %[[T0:.*]] = vector.create_mask %[[B]] : vector<3xi1> // CHECK: %[[T1:.*]] = arith.cmpi sgt, %[[A]], %[[c0]] : index // CHECK: %[[T2:.*]] = arith.select %[[T1]], %[[T0]], %[[C1]] : vector<3xi1> diff --git a/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir b/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir index 17cdda65fb2c..6e06ba1bb14b 100644 --- a/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir @@ -6,9 +6,9 @@ func.func @vector_multi_reduction(%arg0: vector<2x4xf32>, %acc: vector<2xf32>) - } // CHECK-LABEL: func @vector_multi_reduction // CHECK-SAME: %[[INPUT:.+]]: vector<2x4xf32>, %[[ACC:.*]]: vector<2xf32>) -// CHECK: %[[RESULT_VEC_0:.+]] = arith.constant dense<{{.*}}> : vector<2xf32> -// CHECK: %[[C0:.+]] = arith.constant 0 : index -// CHECK: %[[C1:.+]] = arith.constant 1 : index +// CHECK-DAG: %[[RESULT_VEC_0:.+]] = arith.constant dense<{{.*}}> : vector<2xf32> +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index // CHECK: %[[V0:.+]] = vector.extract %[[INPUT]][0] // CHECK: %[[ACC0:.+]] = vector.extract %[[ACC]][0] // CHECK: %[[RV0:.+]] = vector.reduction , %[[V0]], %[[ACC0]] : vector<4xf32> into f32 @@ -37,7 +37,7 @@ func.func @vector_reduction_inner(%arg0: vector<2x3x4x5xi32>, %acc: vector<2x3xi } // CHECK-LABEL: func @vector_reduction_inner // CHECK-SAME: %[[INPUT:.+]]: vector<2x3x4x5xi32>, %[[ACC:.*]]: vector<2x3xi32> -// CHECK: %[[FLAT_RESULT_VEC_0:.+]] = arith.constant dense<0> : vector<6xi32> +// CHECK-DAG: %[[FLAT_RESULT_VEC_0:.+]] = arith.constant dense<0> : vector<6xi32> // CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index // CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index // CHECK-DAG: %[[C2:.+]] = arith.constant 2 : index @@ -90,15 +90,15 @@ func.func @vector_multi_reduction_ordering(%arg0: vector<3x2x4xf32>, %acc: vecto } // CHECK-LABEL: func @vector_multi_reduction_ordering // CHECK-SAME: %[[INPUT:.+]]: vector<3x2x4xf32>, %[[ACC:.*]]: vector<2x4xf32>) -// CHECK: %[[RESULT_VEC_0:.+]] = arith.constant dense<{{.*}}> : vector<8xf32> -// CHECK: %[[C0:.+]] = arith.constant 0 : index -// CHECK: %[[C1:.+]] = arith.constant 1 : index -// CHECK: %[[C2:.+]] = arith.constant 2 : index -// CHECK: %[[C3:.+]] = arith.constant 3 : index -// CHECK: %[[C4:.+]] = arith.constant 4 : index -// CHECK: %[[C5:.+]] = arith.constant 5 : index -// CHECK: %[[C6:.+]] = arith.constant 6 : index -// CHECK: %[[C7:.+]] = arith.constant 7 : index +// CHECK-DAG: %[[RESULT_VEC_0:.+]] = arith.constant dense<{{.*}}> : vector<8xf32> +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.+]] = arith.constant 1 : index +// CHECK-DAG: %[[C2:.+]] = arith.constant 2 : index +// CHECK-DAG: %[[C3:.+]] = arith.constant 3 : index +// CHECK-DAG: %[[C4:.+]] = arith.constant 4 : index +// CHECK-DAG: %[[C5:.+]] = arith.constant 5 : index +// CHECK-DAG: %[[C6:.+]] = arith.constant 6 : index +// CHECK-DAG: %[[C7:.+]] = arith.constant 7 : index // CHECK: %[[TRANSPOSED_INPUT:.+]] = vector.transpose %[[INPUT]], [1, 2, 0] : vector<3x2x4xf32> to vector<2x4x3xf32> // CHECK: %[[V0:.+]] = vector.extract %[[TRANSPOSED_INPUT]][0, 0] // CHECK: %[[ACC0:.+]] = vector.extract %[[ACC]][0, 0] : f32 from vector<2x4xf32> @@ -256,10 +256,10 @@ func.func private @scalable_dims(%A : vector<8x[4]x2xf32>, %B: vector<8x[4]xf32> // CHECK-LABEL: func.func private @scalable_dims( // CHECK-SAME: %[[VAL_0:.*]]: vector<8x[4]x2xf32>, // CHECK-SAME: %[[VAL_1:.*]]: vector<8x[4]xf32>) -> vector<8x[4]xf32> { -// CHECK: %[[VAL_2:.*]] = arith.constant dense<0.000000e+00> : vector<[32]xf32> -// CHECK: %[[VAL_3:.*]] = arith.constant 0 : index -// CHECK: %[[VAL_4:.*]] = arith.constant 1 : index -// CHECK: %[[VAL_34:.*]] = arith.constant 31 : index +// CHECK-DAG: %[[VAL_2:.*]] = arith.constant dense<0.000000e+00> : vector<[32]xf32> +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[VAL_34:.*]] = arith.constant 31 : index // CHECK: %[[VAL_35:.*]] = vector.extract %[[VAL_0]][0, 0] : vector<2xf32> from vector<8x[4]x2xf32> // CHECK: %[[VAL_36:.*]] = vector.extract %[[VAL_1]][0, 0] : f32 from vector<8x[4]xf32> diff --git a/mlir/test/Dialect/Vector/vector-scalable-create-mask-lowering.mlir b/mlir/test/Dialect/Vector/vector-scalable-create-mask-lowering.mlir index 95c77a43155e..772388b6490c 100644 --- a/mlir/test/Dialect/Vector/vector-scalable-create-mask-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-scalable-create-mask-lowering.mlir @@ -2,8 +2,8 @@ // CHECK-LABEL: func.func @create_mask_2d_trailing_scalable( // CHECK-SAME: %[[arg:.*]]: index) -> vector<3x[4]xi1> { -// CHECK-NEXT: %[[zero_mask_1d:.*]] = arith.constant dense : vector<[4]xi1> -// CHECK-NEXT: %[[zero_mask_2d:.*]] = arith.constant dense : vector<3x[4]xi1> +// CHECK-DAG: %[[zero_mask_1d:.*]] = arith.constant dense : vector<[4]xi1> +// CHECK-DAG: %[[zero_mask_2d:.*]] = arith.constant dense : vector<3x[4]xi1> // CHECK-NEXT: %[[create_mask_1d:.*]] = vector.create_mask %[[arg]] : vector<[4]xi1> // CHECK-NEXT: %[[res_0:.*]] = vector.insert %[[create_mask_1d]], %[[zero_mask_2d]] [0] : vector<[4]xi1> into vector<3x[4]xi1> // CHECK-NEXT: %[[res_1:.*]] = vector.insert %[[create_mask_1d]], %[[res_0]] [1] : vector<[4]xi1> into vector<3x[4]xi1> -- GitLab From b662c9aa0e0580e1fc78a8787414c86ad1984742 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Fri, 5 Jan 2024 09:30:52 +0100 Subject: [PATCH 080/728] [mlir][bufferization][NFC] Buffer deallocation: Add comment to `handleInterface` (#76956) This is a follow-up for #68648. --- .../Transforms/OwnershipBasedBufferDeallocation.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp index 9459cc43547f..529d5a808c01 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp @@ -861,6 +861,12 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) { for (auto operand : llvm::make_filter_range(op->getOperands(), isMemref)) { if (op.getEffectOnValue(operand).has_value()) { + // The bufferization.manual_deallocation attribute can be attached to ops + // with an allocation and/or deallocation side effect. It indicates that + // the op is under a "manual deallocation" scheme. Deallocation ops are + // usually forbidden in the input IR (not supported by the buffer + // deallocation pass). However, if they are under manual deallocation, + // they can be safely ignored by the buffer deallocation pass. if (!op->hasAttr(BufferizationDialect::kManualDeallocation)) return op->emitError( "memory free side-effect on MemRef value not supported!"); -- GitLab From 71f56e49ceca75dbf82cbb9537c2545c2d2e51c9 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 5 Jan 2024 09:53:05 +0100 Subject: [PATCH 081/728] [ConstraintElim] Decompose shl nsw for signed predicates (#76961) shl nsw x, shift can be interpreted as mul nsw x, (1<getValue().getLimitedValue(); + if (Shift < Ty->getIntegerBitWidth() - 1) { + assert(Shift < 64 && "Would overflow"); + auto Result = decompose(Op0, Preconditions, IsSigned, DL); + Result.mul(int64_t(1) << Shift); + return Result; + } + } + return V; } diff --git a/llvm/test/Transforms/ConstraintElimination/shl.ll b/llvm/test/Transforms/ConstraintElimination/shl.ll index 8a00eb9b2830..a7cc438fe210 100644 --- a/llvm/test/Transforms/ConstraintElimination/shl.ll +++ b/llvm/test/Transforms/ConstraintElimination/shl.ll @@ -1283,8 +1283,7 @@ define i1 @shl_nsw_x8_slt_x7(i8 %start, i8 %high) { ; CHECK-NEXT: [[C_1:%.*]] = icmp slt i8 [[START_SHL_3]], [[HIGH]] ; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]]) ; CHECK-NEXT: [[START_MUL_7:%.*]] = mul nsw i8 [[START]], 7 -; CHECK-NEXT: [[T_1:%.*]] = icmp slt i8 [[START_MUL_7]], [[HIGH]] -; CHECK-NEXT: ret i1 [[T_1]] +; CHECK-NEXT: ret i1 true ; %c.0 = icmp sge i8 %high, 0 call void @llvm.assume(i1 %c.0) @@ -1327,11 +1326,9 @@ define i1 @shl_nsw_sign_implication(i8 %x) { ; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i8 [[X]], 0 ; CHECK-NEXT: br i1 [[CMP1]], label [[IF:%.*]], label [[ELSE:%.*]] ; CHECK: if: -; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i8 [[SHL]], 0 -; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK-NEXT: ret i1 true ; CHECK: else: -; CHECK-NEXT: [[CMP3:%.*]] = icmp sge i8 [[SHL]], 0 -; CHECK-NEXT: ret i1 [[CMP3]] +; CHECK-NEXT: ret i1 true ; %shl = shl nsw i8 %x, 2 %cmp1 = icmp slt i8 %x, 0 @@ -1361,3 +1358,37 @@ define i1 @shl_nsw_by_bw_minus_1(i64 %x) { %t.1 = icmp slt i64 %x, 0 ret i1 %t.1 } + +; Shift returns poison in this case, just make sure we don't crash. +define i1 @shl_nsw_by_bw(i64 %x) { +; CHECK-LABEL: @shl_nsw_by_bw( +; CHECK-NEXT: [[X_SHL:%.*]] = shl nsw i64 [[X:%.*]], 64 +; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[X_SHL]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]]) +; CHECK-NEXT: [[T_1:%.*]] = icmp slt i64 [[X]], 0 +; CHECK-NEXT: ret i1 [[T_1]] +; + %x.shl = shl nsw i64 %x, 64 + %c.1 = icmp slt i64 %x.shl, 0 + call void @llvm.assume(i1 %c.1) + + %t.1 = icmp slt i64 %x, 0 + ret i1 %t.1 +} + +; Shift returns poison in this case, just make sure we don't crash. +define i1 @shl_nsw_by_bw_plus_1(i64 %x) { +; CHECK-LABEL: @shl_nsw_by_bw_plus_1( +; CHECK-NEXT: [[X_SHL:%.*]] = shl nsw i64 [[X:%.*]], 65 +; CHECK-NEXT: [[C_1:%.*]] = icmp slt i64 [[X_SHL]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]]) +; CHECK-NEXT: [[T_1:%.*]] = icmp slt i64 [[X]], 0 +; CHECK-NEXT: ret i1 [[T_1]] +; + %x.shl = shl nsw i64 %x, 65 + %c.1 = icmp slt i64 %x.shl, 0 + call void @llvm.assume(i1 %c.1) + + %t.1 = icmp slt i64 %x, 0 + ret i1 %t.1 +} -- GitLab From 2a1e3906a4913ebf11760edb8d51c099a9cedf41 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Fri, 5 Jan 2024 08:55:14 +0000 Subject: [PATCH 082/728] [llvm][lit] Handle case when there is no llvm default target triple (#76934) This can happen when you do not choose a specific target triple, and do not enable the host architecture when building (if you do enable it, it would become the default target). Such as only enabling RISC-V, when building on an AArch64 machine. Originally reported https://discourse.llvm.org/t/llvm-test-error-could-not-turn-into-itanium-abi-triple/76013. When attempting to run a single test via lit you get: `Could not turn '' into Itanium ABI triple` Setting a default triple with `LLVM_DEFAULT_TARGET_TRIPLE` works around the issue. This change copies the existing host triple check to target triple, and adds a note to highlight the potential issue. As `check-clang` on my AArch64 machine failed 32% of tests in this configuration. Which is to be expected and is ok if you only want to run specific tests, but for anyone unintentionally building this way the note is a clue to the cause. --- llvm/utils/lit/lit/llvm/config.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py index 8b0f2e6295ad..96b4f7bc8677 100644 --- a/llvm/utils/lit/lit/llvm/config.py +++ b/llvm/utils/lit/lit/llvm/config.py @@ -646,15 +646,25 @@ class LLVMConfig(object): self.add_tool_substitutions(tool_substitutions) self.config.substitutions.append(("%resource_dir", builtin_include_dir)) - self.config.substitutions.append( - ( - "%itanium_abi_triple", - self.make_itanium_abi_triple(self.config.target_triple), + # There will be no default target triple if one was not specifically + # set, and the host's architecture is not an enabled target. + if self.config.target_triple: + self.config.substitutions.append( + ( + "%itanium_abi_triple", + self.make_itanium_abi_triple(self.config.target_triple), + ) ) - ) - self.config.substitutions.append( - ("%ms_abi_triple", self.make_msabi_triple(self.config.target_triple)) - ) + self.config.substitutions.append( + ("%ms_abi_triple", self.make_msabi_triple(self.config.target_triple)) + ) + else: + if not self.lit_config.quiet: + self.lit_config.note( + "No default target triple was found, some tests may fail as a result." + ) + self.config.substitutions.append(("%itanium_abi_triple", "")) + self.config.substitutions.append(("%ms_abi_triple", "")) # The host triple might not be set, at least if we're compiling clang # from an already installed llvm. -- GitLab From 190a75b5f12d3872a5a26d6079d62adae40f147d Mon Sep 17 00:00:00 2001 From: Utkarsh Saxena Date: Fri, 5 Jan 2024 10:07:04 +0100 Subject: [PATCH 083/728] [coroutines] Introduce [[clang::coro_disable_lifetimebound]] (#76818) Lifetime-bound analysis of reference parameters of coroutines and coroutine wrappers is helpful in surfacing memory bugs associated with using temporaries and stack variables in call expressions in plain return statements. This is the default semantics of `[[clang::coro_lifetimebound]]`. But it should be okay to relax the requirements for a function when the reference arguments are not lifetime bound. For example: A coroutine wrapper accepts a reference parameter but does not pass it to the underlying coroutine call. ```cpp [[clang::coro_wrapper]] Task wrapper(const Request& req) { return req.shouldCallA() ? coroA() : coroB(); } ``` Or passes it the coroutine by value ```cpp Task coro(std::string s) { co_return s.size(); } [[clang::coro_wrapper]] wrapper(const std::string& s) { return coro(s); } ``` This patch allows functions to be annotated with `[[clang::coro_disable_lifetime_bound]]` to disable lifetime bound analysis for all calls to this function. --- One missing piece here is a note suggesting using this annotation in cases of lifetime warnings. This would require some more tweaks in the lifetimebound analysis to recognize violations involving coroutines only and produce this note only in those cases. --- clang/docs/ReleaseNotes.rst | 1 + clang/include/clang/Basic/Attr.td | 8 ++++++ clang/include/clang/Basic/AttrDocs.td | 26 ++++++++++++++++--- clang/lib/Sema/SemaInit.cpp | 3 ++- ...a-attribute-supported-attributes-list.test | 1 + clang/test/SemaCXX/coro-lifetimebound.cpp | 15 +++++++++++ 6 files changed, 49 insertions(+), 5 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index ce7599ad34be..903a50a4d4d3 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -357,6 +357,7 @@ Attribute Changes in Clang - Clang now introduced ``[[clang::coro_lifetimebound]]`` attribute. All parameters of a function are considered to be lifetime bound if the function returns a type annotated with ``[[clang::coro_lifetimebound]]`` and ``[[clang::coro_return_type]]``. + This analysis can be disabled for a function by annotating the function with ``[[clang::coro_disable_lifetimebound]]``. Improvements to Clang's diagnostics ----------------------------------- diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index db17211747b1..fda62aaae22c 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -1121,6 +1121,14 @@ def CoroLifetimeBound : InheritableAttr { let SimpleHandler = 1; } +def CoroDisableLifetimeBound : InheritableAttr { + let Spellings = [Clang<"coro_disable_lifetimebound">]; + let Subjects = SubjectList<[Function]>; + let LangOpts = [CPlusPlus]; + let Documentation = [CoroLifetimeBoundDoc]; + let SimpleHandler = 1; +} + // OSObject-based attributes. def OSConsumed : InheritableParamAttr { let Spellings = [Clang<"os_consumed">]; diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index 98a7ecc7fd7d..cd3dcf2ccf44 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -7671,9 +7671,12 @@ The ``[[clang::coro_lifetimebound]]`` is a class attribute which can be applied to a coroutine return type (`CRT`_) (i.e. it should also be annotated with ``[[clang::coro_return_type]]``). -All parameters of a function are considered to be lifetime bound. See `documentation`_ -of ``[[clang::lifetimebound]]`` for more details. -if the function returns a coroutine return type (CRT) annotated with ``[[clang::coro_lifetimebound]]``. +All parameters of a function are considered to be lifetime bound if the function returns a +coroutine return type (CRT) annotated with ``[[clang::coro_lifetimebound]]``. +This lifetime bound analysis can be disabled for a coroutine wrapper or a coroutine by annotating the function +with ``[[clang::coro_disable_lifetimebound]]`` function attribute . +See `documentation`_ of ``[[clang::lifetimebound]]`` for details about lifetime bound analysis. + Reference parameters of a coroutine are susceptible to capturing references to temporaries or local variables. @@ -7703,7 +7706,7 @@ Both coroutines and coroutine wrappers are part of this analysis. }; Task coro(const int& a) { co_return a + 1; } - Task [[clang::coro_wrapper]] coro_wrapper(const int& a, const int& b) { + [[clang::coro_wrapper]] Task coro_wrapper(const int& a, const int& b) { return a > b ? coro(a) : coro(b); } Task temporary_reference() { @@ -7718,6 +7721,21 @@ Both coroutines and coroutine wrappers are part of this analysis. return coro(a); // warning: returning address of stack variable `a`. } +This analysis can be disabled for all calls to a particular function by annotating the function +with function attribute ``[[clang::coro_disable_lifetimebound]]``. +For example, this could be useful for coroutine wrappers which accept reference parameters +but do not pass them to the underlying coroutine or pass them by value. + +.. code-block:: c++ + + Task coro(int a) { co_return a + 1; } + [[clang::coro_wrapper, clang::coro_disable_lifetimebound]] Task coro_wrapper(const int& a) { + return coro(a + 1); + } + void use() { + auto task = coro_wrapper(1); // use of temporary is fine as the argument is not lifetime bound. + } + .. _`documentation`: https://clang.llvm.org/docs/AttributeReference.html#lifetimebound .. _`CRT`: https://clang.llvm.org/docs/AttributeReference.html#coro-return-type }]; diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 61d244f3bb97..60c0e3e74204 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -7581,7 +7581,8 @@ static void visitLifetimeBoundArguments(IndirectLocalPath &Path, Expr *Call, bool CheckCoroCall = false; if (const auto *RD = Callee->getReturnType()->getAsRecordDecl()) { CheckCoroCall = RD->hasAttr() && - RD->hasAttr(); + RD->hasAttr() && + !Callee->hasAttr(); } for (unsigned I = 0, N = std::min(Callee->getNumParams(), Args.size()); diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test index 7b0cda0bca07..2f80c96e1d52 100644 --- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test +++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test @@ -57,6 +57,7 @@ // CHECK-NEXT: ConsumableAutoCast (SubjectMatchRule_record) // CHECK-NEXT: ConsumableSetOnRead (SubjectMatchRule_record) // CHECK-NEXT: Convergent (SubjectMatchRule_function) +// CHECK-NEXT: CoroDisableLifetimeBound (SubjectMatchRule_function) // CHECK-NEXT: CoroLifetimeBound (SubjectMatchRule_record) // CHECK-NEXT: CoroOnlyDestroyWhenComplete (SubjectMatchRule_record) // CHECK-NEXT: CoroReturnType (SubjectMatchRule_record) diff --git a/clang/test/SemaCXX/coro-lifetimebound.cpp b/clang/test/SemaCXX/coro-lifetimebound.cpp index b4dc029a1398..3fc7ca70a14a 100644 --- a/clang/test/SemaCXX/coro-lifetimebound.cpp +++ b/clang/test/SemaCXX/coro-lifetimebound.cpp @@ -115,3 +115,18 @@ CoNoCRT bar(int a) { co_return 1; } } // namespace not_a_crt + +// ============================================================================= +// Not lifetime bound coroutine wrappers: [[clang::coro_disable_lifetimebound]]. +// ============================================================================= +namespace disable_lifetimebound { +Co foo(int x) { co_return x; } + +[[clang::coro_wrapper, clang::coro_disable_lifetimebound]] +Co foo_wrapper(const int& x) { return foo(x); } + +[[clang::coro_wrapper]] Co caller() { + // The call to foo_wrapper is wrapper is safe. + return foo_wrapper(1); +} +} // namespace disable_lifetimebound -- GitLab From 86ef039220de4b7d2e2f1d5e93874ae7a242730e Mon Sep 17 00:00:00 2001 From: Z572 Date: Fri, 5 Jan 2024 03:08:49 -0600 Subject: [PATCH 084/728] [InstCombine] Simplify compare abs(X) and X. (#76385) fix https://github.com/llvm/llvm-project/issues/72653 proof: https://alive2.llvm.org/ce/z/LZzZaj --- .../InstCombine/InstCombineCompares.cpp | 51 ++++ llvm/test/Transforms/InstCombine/icmp-abs.ll | 219 ++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 llvm/test/Transforms/InstCombine/icmp-abs.ll diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 2dc0b0b87f60..a2ff8f3eef81 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -6868,6 +6868,57 @@ Instruction *InstCombinerImpl::foldICmpCommutative(ICmpInst::Predicate Pred, return foldICmpAddOpConst(X, *C, Pred); } + // abs(X) >= X --> true + // abs(X) u<= X --> true + // abs(X) < X --> false + // abs(X) u> X --> false + // abs(X) u>= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN` + // abs(X) <= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN` + // abs(X) == X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN` + // abs(X) u< X --> IsIntMinPosion ? `X < 0` : `X > INTMIN` + // abs(X) > X --> IsIntMinPosion ? `X < 0` : `X > INTMIN` + // abs(X) != X --> IsIntMinPosion ? `X < 0` : `X > INTMIN` + { + Value *X; + Constant *C; + if (match(Op0, m_Intrinsic(m_Value(X), m_Constant(C))) && + match(Op1, m_Specific(X))) { + Value *NullValue = Constant::getNullValue(X->getType()); + Value *AllOnesValue = Constant::getAllOnesValue(X->getType()); + const APInt SMin = + APInt::getSignedMinValue(X->getType()->getScalarSizeInBits()); + bool IsIntMinPosion = C->isAllOnesValue(); + switch (Pred) { + case CmpInst::ICMP_ULE: + case CmpInst::ICMP_SGE: + return replaceInstUsesWith(CxtI, ConstantInt::getTrue(CxtI.getType())); + case CmpInst::ICMP_UGT: + case CmpInst::ICMP_SLT: + return replaceInstUsesWith(CxtI, ConstantInt::getFalse(CxtI.getType())); + case CmpInst::ICMP_UGE: + case CmpInst::ICMP_SLE: + case CmpInst::ICMP_EQ: { + return replaceInstUsesWith( + CxtI, IsIntMinPosion + ? Builder.CreateICmpSGT(X, AllOnesValue) + : Builder.CreateICmpULT( + X, ConstantInt::get(X->getType(), SMin + 1))); + } + case CmpInst::ICMP_ULT: + case CmpInst::ICMP_SGT: + case CmpInst::ICMP_NE: { + return replaceInstUsesWith( + CxtI, IsIntMinPosion + ? Builder.CreateICmpSLT(X, NullValue) + : Builder.CreateICmpUGT( + X, ConstantInt::get(X->getType(), SMin))); + } + default: + llvm_unreachable("Invalid predicate!"); + } + } + } + return nullptr; } diff --git a/llvm/test/Transforms/InstCombine/icmp-abs.ll b/llvm/test/Transforms/InstCombine/icmp-abs.ll new file mode 100644 index 000000000000..f608fb1b914d --- /dev/null +++ b/llvm/test/Transforms/InstCombine/icmp-abs.ll @@ -0,0 +1,219 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +declare i4 @llvm.abs.i4(i4, i1) + +define i1 @icmp_sge_abs(i4 %arg) { +; CHECK-LABEL: @icmp_sge_abs( +; CHECK-NEXT: ret i1 true +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp sge i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_sge_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_sge_abs_false( +; CHECK-NEXT: ret i1 true +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp sge i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_eq_abs(i4 %arg) { +; CHECK-LABEL: @icmp_eq_abs( +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i4 [[ARG:%.*]], -1 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp eq i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_eq_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_eq_abs_false( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i4 [[ARG:%.*]], -7 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp eq i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ne_abs(i4 %arg) { +; CHECK-LABEL: @icmp_ne_abs( +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i4 [[ARG:%.*]], 0 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp ne i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ne_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_ne_abs_false( +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i4 [[ARG:%.*]], -8 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp ne i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_sle_abs(i4 %arg) { +; CHECK-LABEL: @icmp_sle_abs( +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i4 [[ARG:%.*]], -1 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp sle i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_sle_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_sle_abs_false( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i4 [[ARG:%.*]], -7 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp sle i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_slt_abs(i4 %arg) { +; CHECK-LABEL: @icmp_slt_abs( +; CHECK-NEXT: ret i1 false +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp slt i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_slt_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_slt_abs_false( +; CHECK-NEXT: ret i1 false +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp slt i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_sgt_abs(i4 %arg) { +; CHECK-LABEL: @icmp_sgt_abs( +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i4 [[ARG:%.*]], 0 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp sgt i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_sgt_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_sgt_abs_false( +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i4 [[ARG:%.*]], -8 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp sgt i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ugt_abs(i4 %arg) { +; CHECK-LABEL: @icmp_ugt_abs( +; CHECK-NEXT: ret i1 false +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp ugt i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ugt_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_ugt_abs_false( +; CHECK-NEXT: ret i1 false +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp ugt i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_uge_abs(i4 %arg) { +; CHECK-LABEL: @icmp_uge_abs( +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i4 [[ARG:%.*]], -1 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp uge i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_uge_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_uge_abs_false( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i4 [[ARG:%.*]], -7 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp uge i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ule_abs(i4 %arg) { +; CHECK-LABEL: @icmp_ule_abs( +; CHECK-NEXT: ret i1 true +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp ule i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ule_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_ule_abs_false( +; CHECK-NEXT: ret i1 true +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp ule i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ult_abs(i4 %arg) { +; CHECK-LABEL: @icmp_ult_abs( +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i4 [[ARG:%.*]], 0 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp ult i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_ult_abs_false(i4 %arg) { +; CHECK-LABEL: @icmp_ult_abs_false( +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i4 [[ARG:%.*]], -8 +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 false) + %cmp = icmp ult i4 %abs, %arg + ret i1 %cmp +} + +define i1 @icmp_sge_abs2(i4 %arg) { +; CHECK-LABEL: @icmp_sge_abs2( +; CHECK-NEXT: [[X:%.*]] = mul i4 [[ARG:%.*]], [[ARG]] +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i4 [[X]], -1 +; CHECK-NEXT: ret i1 [[CMP]] +; + %x = mul i4 %arg, %arg ; thwart complexity-based canonicalization + %abs = call i4 @llvm.abs.i4(i4 %x, i1 true) + %cmp = icmp sge i4 %x, %abs + ret i1 %cmp +} + +define i1 @icmp_sge_abs_mismatched_op(i4 %arg, i4 %arg2) { +; CHECK-LABEL: @icmp_sge_abs_mismatched_op( +; CHECK-NEXT: [[ABS:%.*]] = call i4 @llvm.abs.i4(i4 [[ARG:%.*]], i1 true) +; CHECK-NEXT: [[CMP:%.*]] = icmp sge i4 [[ABS]], [[ARG2:%.*]] +; CHECK-NEXT: ret i1 [[CMP]] +; + %abs = call i4 @llvm.abs.i4(i4 %arg, i1 true) + %cmp = icmp sge i4 %abs, %arg2 + ret i1 %cmp + } -- GitLab From 67963d384b23a2d46967b8f39ec2df3375731f4f Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Fri, 5 Jan 2024 10:13:33 +0100 Subject: [PATCH 085/728] [include-cleaner] Fix a race issue when editing multiple files. (#76960) We have a previous fix https://github.com/llvm/llvm-project/commit/be861b64d94198230d8f9889b17280e3cd215a0a, which snapshots all processing files. It works most of times, the snapshot (InMemoryFileSystem) is based on the file path. The file-path-based lookup can fail in a subtle way for some tricky cases (we encounter it internally), which will result in reading a corrupted file. This is a different fix, we don't modify files on the fly, instead, we write files when the tool finishes for all files. --- .../include-cleaner/tool/IncludeCleaner.cpp | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp b/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp index 30aaee29b9a3..e078bfae66c3 100644 --- a/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp +++ b/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp @@ -16,10 +16,10 @@ #include "clang/Tooling/Tooling.h" #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Regex.h" #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" @@ -110,14 +110,16 @@ format::FormatStyle getStyle(llvm::StringRef Filename) { class Action : public clang::ASTFrontendAction { public: - Action(llvm::function_ref HeaderFilter) - : HeaderFilter(HeaderFilter){}; + Action(llvm::function_ref HeaderFilter, + llvm::StringMap &EditedFiles) + : HeaderFilter(HeaderFilter), EditedFiles(EditedFiles) {} private: RecordedAST AST; RecordedPP PP; PragmaIncludes PI; llvm::function_ref HeaderFilter; + llvm::StringMap &EditedFiles; bool BeginInvocation(CompilerInstance &CI) override { // We only perform include-cleaner analysis. So we disable diagnostics that @@ -181,17 +183,8 @@ private: } } - if (Edit && (!Results.Missing.empty() || !Results.Unused.empty())) { - if (auto Err = llvm::writeToOutput( - Path, [&](llvm::raw_ostream &OS) -> llvm::Error { - OS << Final; - return llvm::Error::success(); - })) { - llvm::errs() << "Failed to apply edits to " << Path << ": " - << toString(std::move(Err)) << "\n"; - ++Errors; - } - } + if (!Results.Missing.empty() || !Results.Unused.empty()) + EditedFiles.try_emplace(Path, Final); } void writeHTML() { @@ -215,11 +208,17 @@ public: : HeaderFilter(HeaderFilter) {} std::unique_ptr create() override { - return std::make_unique(HeaderFilter); + return std::make_unique(HeaderFilter, EditedFiles); + } + + const llvm::StringMap &editedFiles() const { + return EditedFiles; } private: llvm::function_ref HeaderFilter; + // Map from file name to final code with the include edits applied. + llvm::StringMap EditedFiles; }; std::function headerFilter() { @@ -274,21 +273,26 @@ int main(int argc, const char **argv) { clang::tooling::ClangTool Tool(OptionsParser->getCompilations(), OptionsParser->getSourcePathList()); - std::vector> Buffers; - for (const auto &File : OptionsParser->getSourcePathList()) { - auto Content = llvm::MemoryBuffer::getFile(File); - if (!Content) { - llvm::errs() << "Error: can't read file '" << File - << "': " << Content.getError().message() << "\n"; - return 1; - } - Buffers.push_back(std::move(Content.get())); - Tool.mapVirtualFile(File, Buffers.back()->getBuffer()); - } auto HeaderFilter = headerFilter(); if (!HeaderFilter) return 1; // error already reported. ActionFactory Factory(HeaderFilter); - return Tool.run(&Factory) || Errors != 0; + auto ErrorCode = Tool.run(&Factory); + if (Edit) { + for (const auto &NameAndContent : Factory.editedFiles()) { + llvm::StringRef FileName = NameAndContent.first(); + const std::string &FinalCode = NameAndContent.second; + if (auto Err = llvm::writeToOutput( + FileName, [&](llvm::raw_ostream &OS) -> llvm::Error { + OS << FinalCode; + return llvm::Error::success(); + })) { + llvm::errs() << "Failed to apply edits to " << FileName << ": " + << toString(std::move(Err)) << "\n"; + ++Errors; + } + } + } + return ErrorCode || Errors != 0; } -- GitLab From 4daea501c4fc969bc6d8baafe646487ae1881aab Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Fri, 5 Jan 2024 17:16:55 +0800 Subject: [PATCH 086/728] [X86][MC] Support encoding/decoding for APX variant MUL/IMUL/DIV/IDIV instructions (#76919) Four variants: promoted legacy, ND (new data destination), NF (no flags update) and NF_ND (NF + ND). The syntax of NF instructions is aligned with GNU binutils. https://sourceware.org/pipermail/binutils/2023-September/129545.html --- llvm/lib/Target/X86/X86InstrArithmetic.td | 283 ++++++++++++++++----- llvm/lib/Target/X86/X86InstrPredicates.td | 33 +++ llvm/test/MC/Disassembler/X86/apx/div.txt | 66 +++++ llvm/test/MC/Disassembler/X86/apx/idiv.txt | 66 +++++ llvm/test/MC/Disassembler/X86/apx/imul.txt | 258 +++++++++++++++++++ llvm/test/MC/Disassembler/X86/apx/mul.txt | 66 +++++ llvm/test/MC/X86/apx/div-att.s | 53 ++++ llvm/test/MC/X86/apx/div-intel.s | 50 ++++ llvm/test/MC/X86/apx/idiv-att.s | 53 ++++ llvm/test/MC/X86/apx/idiv-intel.s | 50 ++++ llvm/test/MC/X86/apx/imul-att.s | 197 ++++++++++++++ llvm/test/MC/X86/apx/imul-intel.s | 194 ++++++++++++++ llvm/test/MC/X86/apx/mul-att.s | 53 ++++ llvm/test/MC/X86/apx/mul-intel.s | 50 ++++ llvm/test/TableGen/x86-fold-tables.inc | 31 +++ 15 files changed, 1438 insertions(+), 65 deletions(-) create mode 100644 llvm/test/MC/Disassembler/X86/apx/div.txt create mode 100644 llvm/test/MC/Disassembler/X86/apx/idiv.txt create mode 100644 llvm/test/MC/Disassembler/X86/apx/imul.txt create mode 100644 llvm/test/MC/Disassembler/X86/apx/mul.txt create mode 100644 llvm/test/MC/X86/apx/div-att.s create mode 100644 llvm/test/MC/X86/apx/div-intel.s create mode 100644 llvm/test/MC/X86/apx/idiv-att.s create mode 100644 llvm/test/MC/X86/apx/idiv-intel.s create mode 100644 llvm/test/MC/X86/apx/imul-att.s create mode 100644 llvm/test/MC/X86/apx/imul-intel.s create mode 100644 llvm/test/MC/X86/apx/mul-att.s create mode 100644 llvm/test/MC/X86/apx/mul-intel.s diff --git a/llvm/lib/Target/X86/X86InstrArithmetic.td b/llvm/lib/Target/X86/X86InstrArithmetic.td index ed9f45bdd9d1..5cfa95e085e3 100644 --- a/llvm/lib/Target/X86/X86InstrArithmetic.td +++ b/llvm/lib/Target/X86/X86InstrArithmetic.td @@ -71,24 +71,60 @@ multiclass Mul o, string m, Format RegMRM, Format MemMRM, SDPatternOpera // FIXME: Used for 8-bit mul, ignore result upper 8 bits. // This probably ought to be moved to a def : Pat<> if the // syntax can be accepted. - let Defs = [AL,EFLAGS,AX], Uses = [AL] in - def 8r : MulDivOpR; - let Defs = [AX,DX,EFLAGS], Uses = [AX] in - def 16r : MulDivOpR, OpSize16; - let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in - def 32r : MulDivOpR, OpSize32; - let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in - def 64r : MulDivOpR; - let Defs = [AL,EFLAGS,AX], Uses = [AL] in - def 8m : MulDivOpM; - let Defs = [AX,DX,EFLAGS], Uses = [AX] in - def 16m : MulDivOpM, OpSize16; - let Defs = [EAX,EDX,EFLAGS], Uses = [EAX] in - def 32m : MulDivOpM, OpSize32; - let Defs = [RAX,RDX,EFLAGS], Uses = [RAX] in - def 64m : MulDivOpM, Requires<[In64BitMode]>; + let Defs = [AL, EFLAGS, AX], Uses = [AL] in + def 8r : MulDivOpR; + let Defs = [AX, DX, EFLAGS], Uses = [AX] in + def 16r : MulDivOpR, OpSize16; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in + def 32r : MulDivOpR, OpSize32; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX] in + def 64r : MulDivOpR; + let Defs = [AL, EFLAGS, AX], Uses = [AL] in + def 8m : MulDivOpM; + let Defs = [AX, DX, EFLAGS], Uses = [AX] in + def 16m : MulDivOpM, OpSize16; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in + def 32m : MulDivOpM, OpSize32; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX] in + def 64m : MulDivOpM, Requires<[In64BitMode]>; + + let Predicates = [In64BitMode] in { + let Defs = [AL, AX], Uses = [AL] in + def 8r_NF : MulDivOpR, NF; + let Defs = [AX, DX], Uses = [AX] in + def 16r_NF : MulDivOpR, NF, PD; + let Defs = [EAX, EDX], Uses = [EAX] in + def 32r_NF : MulDivOpR, NF; + let Defs = [RAX, RDX], Uses = [RAX] in + def 64r_NF : MulDivOpR, NF; + let Defs = [AL, AX], Uses = [AL] in + def 8m_NF : MulDivOpM, NF; + let Defs = [AX, DX], Uses = [AX] in + def 16m_NF : MulDivOpM, NF, PD; + let Defs = [EAX, EDX], Uses = [EAX] in + def 32m_NF : MulDivOpM, NF; + let Defs = [RAX, RDX], Uses = [RAX] in + def 64m_NF : MulDivOpM, NF; + + let Defs = [AL, EFLAGS, AX], Uses = [AL] in + def 8r_EVEX : MulDivOpR, PL; + let Defs = [AX, DX, EFLAGS], Uses = [AX] in + def 16r_EVEX : MulDivOpR, PL, PD; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in + def 32r_EVEX : MulDivOpR, PL; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX] in + def 64r_EVEX : MulDivOpR, PL; + let Defs = [AL, EFLAGS, AX], Uses = [AL] in + def 8m_EVEX : MulDivOpM, PL; + let Defs = [AX, DX, EFLAGS], Uses = [AX] in + def 16m_EVEX : MulDivOpM, PL, PD; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in + def 32m_EVEX : MulDivOpM, PL; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX] in + def 64m_EVEX : MulDivOpM, PL; + } } defm MUL : Mul<0xF7, "mul", MRM4r, MRM4m, mul>; @@ -99,54 +135,141 @@ multiclass Div o, string m, Format RegMRM, Format MemMRM> { defvar sched16 = !if(!eq(m, "div"), WriteDiv16, WriteIDiv16); defvar sched32 = !if(!eq(m, "div"), WriteDiv32, WriteIDiv32); defvar sched64 = !if(!eq(m, "div"), WriteDiv64, WriteIDiv64); - let Defs = [AL,AH,EFLAGS], Uses = [AX] in - def 8r : MulDivOpR; - let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in - def 16r : MulDivOpR, OpSize16; - let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in - def 32r : MulDivOpR, OpSize32; - let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in - def 64r : MulDivOpR; - let Defs = [AL,AH,EFLAGS], Uses = [AX] in - def 8m : MulDivOpM; - let Defs = [AX,DX,EFLAGS], Uses = [AX,DX] in - def 16m : MulDivOpM, OpSize16; - let Defs = [EAX,EDX,EFLAGS], Uses = [EAX,EDX] in - def 32m : MulDivOpM, OpSize32; - let Defs = [RAX,RDX,EFLAGS], Uses = [RAX,RDX] in - def 64m : MulDivOpM, Requires<[In64BitMode]>; + let Defs = [AL, AH, EFLAGS], Uses = [AX] in + def 8r : MulDivOpR; + let Defs = [AX, DX, EFLAGS], Uses = [AX, DX] in + def 16r : MulDivOpR, OpSize16; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EDX] in + def 32r : MulDivOpR, OpSize32; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RDX] in + def 64r : MulDivOpR; + let Defs = [AL, AH, EFLAGS], Uses = [AX] in + def 8m : MulDivOpM; + let Defs = [AX, DX, EFLAGS], Uses = [AX, DX] in + def 16m : MulDivOpM, OpSize16; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EDX] in + def 32m : MulDivOpM, OpSize32; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RDX] in + def 64m : MulDivOpM, Requires<[In64BitMode]>; + + let Predicates = [In64BitMode] in { + let Defs = [AL, AH], Uses = [AX] in + def 8r_NF : MulDivOpR, NF; + let Defs = [AX, DX], Uses = [AX, DX] in + def 16r_NF : MulDivOpR, NF, PD; + let Defs = [EAX, EDX], Uses = [EAX, EDX] in + def 32r_NF : MulDivOpR, NF; + let Defs = [RAX, RDX], Uses = [RAX, RDX] in + def 64r_NF : MulDivOpR, NF; + let Defs = [AL, AH], Uses = [AX] in + def 8m_NF : MulDivOpM, NF; + let Defs = [AX, DX], Uses = [AX, DX] in + def 16m_NF : MulDivOpM, NF, PD; + let Defs = [EAX, EDX], Uses = [EAX, EDX] in + def 32m_NF : MulDivOpM, NF; + let Defs = [RAX, RDX], Uses = [RAX, RDX] in + def 64m_NF : MulDivOpM, NF; + + let Defs = [AL, AH, EFLAGS], Uses = [AX] in + def 8r_EVEX : MulDivOpR, PL; + let Defs = [AX, DX, EFLAGS], Uses = [AX, DX] in + def 16r_EVEX : MulDivOpR, PL, PD; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EDX] in + def 32r_EVEX : MulDivOpR, PL; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RDX] in + def 64r_EVEX : MulDivOpR, PL; + let Defs = [AL, AH, EFLAGS], Uses = [AX] in + def 8m_EVEX : MulDivOpM, PL; + let Defs = [AX, DX, EFLAGS], Uses = [AX, DX] in + def 16m_EVEX : MulDivOpM, PL, PD; + let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EDX] in + def 32m_EVEX : MulDivOpM, PL; + let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RDX] in + def 64m_EVEX : MulDivOpM, PL; + } } + let hasSideEffects = 1 in { // so that we don't speculatively execute -defm DIV: Div<0xF7, "div", MRM6r, MRM6m>; -defm IDIV: Div<0xF7, "idiv", MRM7r, MRM7m>; + defm DIV: Div<0xF7, "div", MRM6r, MRM6m>; + defm IDIV: Div<0xF7, "idiv", MRM7r, MRM7m>; } -class IMulOpRR - : BinOpRR_RF<0xAF, "imul", t, X86smul_flag>, TB { +class IMulOpRR_R + : BinOpRR_R<0xAF, "imul", t, ndd> { let Form = MRMSrcReg; let SchedRW = [sched]; // X = IMUL Y, Z --> X = IMUL Z, Y let isCommutable = 1; } -class IMulOpRM - : BinOpRM_RF<0xAF, "imul", t, X86smul_flag>, TB { -let Form = MRMSrcMem; -let SchedRW = [sched.Folded, sched.ReadAfterFold]; +class IMulOpRR_RF + : BinOpRR_RF<0xAF, "imul", t, X86smul_flag, ndd> { + let Form = MRMSrcReg; + let SchedRW = [sched]; + // X = IMUL Y, Z --> X = IMUL Z, Y + let isCommutable = 1; +} +class IMulOpRM_R + : BinOpRM_R<0xAF, "imul", t, ndd> { + let Form = MRMSrcMem; + let SchedRW = [sched.Folded, sched.ReadAfterFold]; +} +class IMulOpRM_RF + : BinOpRM_RF<0xAF, "imul", t, X86smul_flag, ndd> { + let Form = MRMSrcMem; + let SchedRW = [sched.Folded, sched.ReadAfterFold]; } -def IMUL16rr : IMulOpRR, OpSize16; -def IMUL32rr : IMulOpRR, OpSize32; -def IMUL64rr : IMulOpRR; -def IMUL16rm : IMulOpRM, OpSize16; -def IMUL32rm : IMulOpRM, OpSize32; -def IMUL64rm : IMulOpRM; +let Predicates = [NoNDD] in { + def IMUL16rr : IMulOpRR_RF, TB, OpSize16; + def IMUL32rr : IMulOpRR_RF, TB, OpSize32; + def IMUL64rr : IMulOpRR_RF, TB; + def IMUL16rm : IMulOpRM_RF, TB, OpSize16; + def IMUL32rm : IMulOpRM_RF, TB, OpSize32; + def IMUL64rm : IMulOpRM_RF, TB; +} +let Predicates = [HasNDD, In64BitMode] in { + def IMUL16rr_ND : IMulOpRR_RF, PD; + def IMUL32rr_ND : IMulOpRR_RF; + def IMUL64rr_ND : IMulOpRR_RF; + def IMUL16rm_ND : IMulOpRM_RF, PD; + def IMUL32rm_ND : IMulOpRM_RF; + def IMUL64rm_ND : IMulOpRM_RF; +} + +let Predicates = [In64BitMode], Pattern = [(null_frag)] in { + def IMUL16rr_NF : IMulOpRR_R, NF, PD; + def IMUL32rr_NF : IMulOpRR_R, NF; + def IMUL64rr_NF : IMulOpRR_R, NF; + def IMUL16rm_NF : IMulOpRM_R, NF, PD; + def IMUL32rm_NF : IMulOpRM_R, NF; + def IMUL64rm_NF : IMulOpRM_R, NF; + + def IMUL16rr_NF_ND : IMulOpRR_R, EVEX_NF, PD; + def IMUL32rr_NF_ND : IMulOpRR_R, EVEX_NF; + def IMUL64rr_NF_ND : IMulOpRR_R, EVEX_NF; + def IMUL16rm_NF_ND : IMulOpRM_R, EVEX_NF, PD; + def IMUL32rm_NF_ND : IMulOpRM_R, EVEX_NF; + def IMUL64rm_NF_ND : IMulOpRM_R, EVEX_NF; + + def IMUL16rr_EVEX : IMulOpRR_RF, PL, PD; + def IMUL32rr_EVEX : IMulOpRR_RF, PL; + def IMUL64rr_EVEX : IMulOpRR_RF, PL; + def IMUL16rm_EVEX : IMulOpRM_RF, PL, PD; + def IMUL32rm_EVEX : IMulOpRM_RF, PL; + def IMUL64rm_EVEX : IMulOpRM_RF, PL; +} class IMulOpRI8_R : BinOpRI8<0x6B, "imul", binop_ndd_args, t, MRMSrcReg, - (outs t.RegClass:$dst)>, DefEFLAGS { + (outs t.RegClass:$dst)> { let SchedRW = [sched]; } class IMulOpRI_R + : BinOpRI<0x69, "imul", binop_ndd_args, t, MRMSrcReg, + (outs t.RegClass:$dst), []> { + let SchedRW = [sched]; +} +class IMulOpRI_RF : BinOpRI<0x69, "imul", binop_ndd_args, t, MRMSrcReg, (outs t.RegClass:$dst), [(set t.RegClass:$dst, EFLAGS, (X86smul_flag t.RegClass:$src1, @@ -154,12 +277,16 @@ class IMulOpRI_R let SchedRW = [sched]; } class IMulOpMI8_R - : BinOpMI8<"imul", binop_ndd_args, t, MRMSrcMem, (outs t.RegClass:$dst)>, - DefEFLAGS { + : BinOpMI8<"imul", binop_ndd_args, t, MRMSrcMem, (outs t.RegClass:$dst)> { let Opcode = 0x6B; let SchedRW = [sched.Folded]; } class IMulOpMI_R + : BinOpMI<0x69, "imul", binop_ndd_args, t, MRMSrcMem, + (outs t.RegClass:$dst), []> { + let SchedRW = [sched.Folded]; +} +class IMulOpMI_RF : BinOpMI<0x69, "imul", binop_ndd_args, t, MRMSrcMem, (outs t.RegClass:$dst), [(set t.RegClass:$dst, EFLAGS, (X86smul_flag (t.LoadNode addr:$src1), @@ -167,20 +294,46 @@ class IMulOpMI_R DefEFLAGS { let SchedRW = [sched.Folded]; } -def IMUL16rri8 : IMulOpRI8_R, OpSize16; -def IMUL32rri8 : IMulOpRI8_R, OpSize32; -def IMUL64rri8 : IMulOpRI8_R; -def IMUL16rri : IMulOpRI_R, OpSize16; -def IMUL32rri : IMulOpRI_R, OpSize32; -def IMUL64rri32 : IMulOpRI_R; - -def IMUL16rmi8 : IMulOpMI8_R, OpSize16; -def IMUL32rmi8 : IMulOpMI8_R, OpSize32; -def IMUL64rmi8 : IMulOpMI8_R; -def IMUL16rmi : IMulOpMI_R, OpSize16; -def IMUL32rmi : IMulOpMI_R, OpSize32; -def IMUL64rmi32 : IMulOpMI_R; - +def IMUL16rri8 : IMulOpRI8_R, DefEFLAGS, OpSize16; +def IMUL32rri8 : IMulOpRI8_R, DefEFLAGS, OpSize32; +def IMUL64rri8 : IMulOpRI8_R, DefEFLAGS; +def IMUL16rri : IMulOpRI_RF, OpSize16; +def IMUL32rri : IMulOpRI_RF, OpSize32; +def IMUL64rri32 : IMulOpRI_RF; +def IMUL16rmi8 : IMulOpMI8_R, DefEFLAGS, OpSize16; +def IMUL32rmi8 : IMulOpMI8_R, DefEFLAGS, OpSize32; +def IMUL64rmi8 : IMulOpMI8_R, DefEFLAGS; +def IMUL16rmi : IMulOpMI_RF, OpSize16; +def IMUL32rmi : IMulOpMI_RF, OpSize32; +def IMUL64rmi32 : IMulOpMI_RF; + +let Predicates = [In64BitMode] in { + def IMUL16rri8_NF : IMulOpRI8_R, NF, PD; + def IMUL32rri8_NF : IMulOpRI8_R, NF; + def IMUL64rri8_NF : IMulOpRI8_R, NF; + def IMUL16rri_NF : IMulOpRI_R, NF, PD; + def IMUL32rri_NF : IMulOpRI_R, NF; + def IMUL64rri32_NF : IMulOpRI_R, NF; + def IMUL16rmi8_NF : IMulOpMI8_R, NF, PD; + def IMUL32rmi8_NF : IMulOpMI8_R, NF; + def IMUL64rmi8_NF : IMulOpMI8_R, NF; + def IMUL16rmi_NF : IMulOpMI_R, NF, PD; + def IMUL32rmi_NF : IMulOpMI_R, NF; + def IMUL64rmi32_NF : IMulOpMI_R, NF; + + def IMUL16rri8_EVEX : IMulOpRI8_R, DefEFLAGS, PL, PD; + def IMUL32rri8_EVEX : IMulOpRI8_R, DefEFLAGS, PL; + def IMUL64rri8_EVEX : IMulOpRI8_R, DefEFLAGS, PL; + def IMUL16rri_EVEX : IMulOpRI_RF, PL, PD; + def IMUL32rri_EVEX : IMulOpRI_RF, PL; + def IMUL64rri32_EVEX : IMulOpRI_RF, PL; + def IMUL16rmi8_EVEX : IMulOpMI8_R, DefEFLAGS, PL, PD; + def IMUL32rmi8_EVEX : IMulOpMI8_R, DefEFLAGS, PL; + def IMUL64rmi8_EVEX : IMulOpMI8_R, DefEFLAGS, PL; + def IMUL16rmi_EVEX : IMulOpMI_RF, PL, PD; + def IMUL32rmi_EVEX : IMulOpMI_RF, PL; + def IMUL64rmi32_EVEX : IMulOpMI_RF, PL; +} //===----------------------------------------------------------------------===// // INC and DEC Instructions // diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td index 94fa6e45ded9..cb751639a057 100644 --- a/llvm/lib/Target/X86/X86InstrPredicates.td +++ b/llvm/lib/Target/X86/X86InstrPredicates.td @@ -8,8 +8,41 @@ def TruePredicate : Predicate<"true">; +// Intel x86 instructions have three separate encoding spaces: legacy, VEX, and +// EVEX. Not all X86 instructions are extended for EGPR. The following is an +// overview of which instructions are extended and how we implement them. +// +// * Legacy space +// All instructions in legacy maps 0 and 1 that have explicit GPR or memory +// operands can use the REX2 prefix to access the EGPR, except XSAVE*/XRSTOR. +// +// * EVEX space +// All instructions in the EVEX space can access the EGPR in their +// register/memory operands. +// +// For the above intructions, the only difference in encoding is reflected in +// the REX2/EVEX prefix when EGPR is used, i.e. the opcode and opcode name are +// unchanged. We don’t add new entries in TD, and instead we extend GPR with +// R16-R31 and make them allocatable only when the feature EGPR is available. +// +// Besides, some instructions in legacy space with map 2/3 and VEX space are +// promoted into EVEX space. Encoding space changes after the promotion, opcode +// and opcode map may change too sometimes. For these instructions, we add new +// entries in TD to avoid overcomplicating the assembler and disassembler. +// +// HasEGPR is for the new entries and NoEGPR is for the entries before +// promotion, so that the promoted variant can be selected first to benefit RA. def HasEGPR : Predicate<"Subtarget->hasEGPR()">; def NoEGPR : Predicate<"!Subtarget->hasEGPR()">; + +// APX extends some instructions with a new form that has an extra register +// operand called a new data destination (NDD). In such forms, NDD is the new +// destination register receiving the result of the computation and all other +// operands (including the original destination operand) become read-only source +// operands. +// +// HasNDD is for the new NDD entries and NoNDD is for the legacy 2-address +// entries, so that the NDD variant can be selected first to benefit RA. def HasNDD : Predicate<"Subtarget->hasNDD()">; def NoNDD : Predicate<"!Subtarget->hasNDD()">; def HasCMOV : Predicate<"Subtarget->canUseCMOV()">; diff --git a/llvm/test/MC/Disassembler/X86/apx/div.txt b/llvm/test/MC/Disassembler/X86/apx/div.txt new file mode 100644 index 000000000000..277e4464c578 --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/div.txt @@ -0,0 +1,66 @@ +# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT +# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL + +# ATT: {evex} divb %bl +# INTEL: {evex} div bl +0x62,0xf4,0x7c,0x08,0xf6,0xf3 + +# ATT: {nf} divb %bl +# INTEL: {nf} div bl +0x62,0xf4,0x7c,0x0c,0xf6,0xf3 + +# ATT: {evex} divw %dx +# INTEL: {evex} div dx +0x62,0xf4,0x7d,0x08,0xf7,0xf2 + +# ATT: {nf} divw %dx +# INTEL: {nf} div dx +0x62,0xf4,0x7d,0x0c,0xf7,0xf2 + +# ATT: {evex} divl %ecx +# INTEL: {evex} div ecx +0x62,0xf4,0x7c,0x08,0xf7,0xf1 + +# ATT: {nf} divl %ecx +# INTEL: {nf} div ecx +0x62,0xf4,0x7c,0x0c,0xf7,0xf1 + +# ATT: {evex} divq %r9 +# INTEL: {evex} div r9 +0x62,0xd4,0xfc,0x08,0xf7,0xf1 + +# ATT: {nf} divq %r9 +# INTEL: {nf} div r9 +0x62,0xd4,0xfc,0x0c,0xf7,0xf1 + +# ATT: {evex} divb 291(%r8,%rax,4) +# INTEL: {evex} div byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf6,0xb4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} divb 291(%r8,%rax,4) +# INTEL: {nf} div byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf6,0xb4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} divw 291(%r8,%rax,4) +# INTEL: {evex} div word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} divw 291(%r8,%rax,4) +# INTEL: {nf} div word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} divl 291(%r8,%rax,4) +# INTEL: {evex} div dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} divl 291(%r8,%rax,4) +# INTEL: {nf} div dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} divq 291(%r8,%rax,4) +# INTEL: {evex} div qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} divq 291(%r8,%rax,4) +# INTEL: {nf} div qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/X86/apx/idiv.txt b/llvm/test/MC/Disassembler/X86/apx/idiv.txt new file mode 100644 index 000000000000..f5daf735a0b5 --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/idiv.txt @@ -0,0 +1,66 @@ +# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT +# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL + +# ATT: {evex} idivb %bl +# INTEL: {evex} idiv bl +0x62,0xf4,0x7c,0x08,0xf6,0xfb + +# ATT: {nf} idivb %bl +# INTEL: {nf} idiv bl +0x62,0xf4,0x7c,0x0c,0xf6,0xfb + +# ATT: {evex} idivw %dx +# INTEL: {evex} idiv dx +0x62,0xf4,0x7d,0x08,0xf7,0xfa + +# ATT: {nf} idivw %dx +# INTEL: {nf} idiv dx +0x62,0xf4,0x7d,0x0c,0xf7,0xfa + +# ATT: {evex} idivl %ecx +# INTEL: {evex} idiv ecx +0x62,0xf4,0x7c,0x08,0xf7,0xf9 + +# ATT: {nf} idivl %ecx +# INTEL: {nf} idiv ecx +0x62,0xf4,0x7c,0x0c,0xf7,0xf9 + +# ATT: {evex} idivq %r9 +# INTEL: {evex} idiv r9 +0x62,0xd4,0xfc,0x08,0xf7,0xf9 + +# ATT: {nf} idivq %r9 +# INTEL: {nf} idiv r9 +0x62,0xd4,0xfc,0x0c,0xf7,0xf9 + +# ATT: {evex} idivb 291(%r8,%rax,4) +# INTEL: {evex} idiv byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf6,0xbc,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} idivb 291(%r8,%rax,4) +# INTEL: {nf} idiv byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf6,0xbc,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} idivw 291(%r8,%rax,4) +# INTEL: {evex} idiv word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} idivw 291(%r8,%rax,4) +# INTEL: {nf} idiv word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} idivl 291(%r8,%rax,4) +# INTEL: {evex} idiv dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} idivl 291(%r8,%rax,4) +# INTEL: {nf} idiv dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} idivq 291(%r8,%rax,4) +# INTEL: {evex} idiv qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} idivq 291(%r8,%rax,4) +# INTEL: {nf} idiv qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/X86/apx/imul.txt b/llvm/test/MC/Disassembler/X86/apx/imul.txt new file mode 100644 index 000000000000..f72a6f4aec8f --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/imul.txt @@ -0,0 +1,258 @@ +# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT +# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL + +# ATT: {evex} imulw $123, %dx, %dx +# INTEL: {evex} imul dx, dx, 123 +0x62,0xf4,0x7d,0x08,0x6b,0xd2,0x7b + +# ATT: {nf} imulw $123, %dx, %dx +# INTEL: {nf} imul dx, dx, 123 +0x62,0xf4,0x7d,0x0c,0x6b,0xd2,0x7b + +# ATT: {evex} imull $123, %ecx, %ecx +# INTEL: {evex} imul ecx, ecx, 123 +0x62,0xf4,0x7c,0x08,0x6b,0xc9,0x7b + +# ATT: {nf} imull $123, %ecx, %ecx +# INTEL: {nf} imul ecx, ecx, 123 +0x62,0xf4,0x7c,0x0c,0x6b,0xc9,0x7b + +# ATT: {evex} imulq $123, %r9, %r9 +# INTEL: {evex} imul r9, r9, 123 +0x62,0x54,0xfc,0x08,0x6b,0xc9,0x7b + +# ATT: {nf} imulq $123, %r9, %r9 +# INTEL: {nf} imul r9, r9, 123 +0x62,0x54,0xfc,0x0c,0x6b,0xc9,0x7b + +# ATT: {evex} imulw $123, 291(%r8,%rax,4), %dx +# INTEL: {evex} imul dx, word ptr [r8 + 4*rax + 291], 123 +0x62,0xd4,0x7d,0x08,0x6b,0x94,0x80,0x23,0x01,0x00,0x00,0x7b + +# ATT: {nf} imulw $123, 291(%r8,%rax,4), %dx +# INTEL: {nf} imul dx, word ptr [r8 + 4*rax + 291], 123 +0x62,0xd4,0x7d,0x0c,0x6b,0x94,0x80,0x23,0x01,0x00,0x00,0x7b + +# ATT: {evex} imull $123, 291(%r8,%rax,4), %ecx +# INTEL: {evex} imul ecx, dword ptr [r8 + 4*rax + 291], 123 +0x62,0xd4,0x7c,0x08,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b + +# ATT: {nf} imull $123, 291(%r8,%rax,4), %ecx +# INTEL: {nf} imul ecx, dword ptr [r8 + 4*rax + 291], 123 +0x62,0xd4,0x7c,0x0c,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b + +# ATT: {evex} imulq $123, 291(%r8,%rax,4), %r9 +# INTEL: {evex} imul r9, qword ptr [r8 + 4*rax + 291], 123 +0x62,0x54,0xfc,0x08,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b + +# ATT: {nf} imulq $123, 291(%r8,%rax,4), %r9 +# INTEL: {nf} imul r9, qword ptr [r8 + 4*rax + 291], 123 +0x62,0x54,0xfc,0x0c,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b + +# ATT: {evex} imulw $1234, %dx, %dx +# INTEL: {evex} imul dx, dx, 1234 +0x62,0xf4,0x7d,0x08,0x69,0xd2,0xd2,0x04 + +# ATT: {nf} imulw $1234, %dx, %dx +# INTEL: {nf} imul dx, dx, 1234 +0x62,0xf4,0x7d,0x0c,0x69,0xd2,0xd2,0x04 + +# ATT: {evex} imulw $1234, 291(%r8,%rax,4), %dx +# INTEL: {evex} imul dx, word ptr [r8 + 4*rax + 291], 1234 +0x62,0xd4,0x7d,0x08,0x69,0x94,0x80,0x23,0x01,0x00,0x00,0xd2,0x04 + +# ATT: {nf} imulw $1234, 291(%r8,%rax,4), %dx +# INTEL: {nf} imul dx, word ptr [r8 + 4*rax + 291], 1234 +0x62,0xd4,0x7d,0x0c,0x69,0x94,0x80,0x23,0x01,0x00,0x00,0xd2,0x04 + +# ATT: {evex} imull $123456, %ecx, %ecx +# INTEL: {evex} imul ecx, ecx, 123456 +0x62,0xf4,0x7c,0x08,0x69,0xc9,0x40,0xe2,0x01,0x00 + +# ATT: {nf} imull $123456, %ecx, %ecx +# INTEL: {nf} imul ecx, ecx, 123456 +0x62,0xf4,0x7c,0x0c,0x69,0xc9,0x40,0xe2,0x01,0x00 + +# ATT: {evex} imulq $123456, %r9, %r9 +# INTEL: {evex} imul r9, r9, 123456 +0x62,0x54,0xfc,0x08,0x69,0xc9,0x40,0xe2,0x01,0x00 + +# ATT: {nf} imulq $123456, %r9, %r9 +# INTEL: {nf} imul r9, r9, 123456 +0x62,0x54,0xfc,0x0c,0x69,0xc9,0x40,0xe2,0x01,0x00 + +# ATT: {evex} imull $123456, 291(%r8,%rax,4), %ecx +# INTEL: {evex} imul ecx, dword ptr [r8 + 4*rax + 291], 123456 +0x62,0xd4,0x7c,0x08,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00 + +# ATT: {nf} imull $123456, 291(%r8,%rax,4), %ecx +# INTEL: {nf} imul ecx, dword ptr [r8 + 4*rax + 291], 123456 +0x62,0xd4,0x7c,0x0c,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00 + +# ATT: {evex} imulq $123456, 291(%r8,%rax,4), %r9 +# INTEL: {evex} imul r9, qword ptr [r8 + 4*rax + 291], 123456 +0x62,0x54,0xfc,0x08,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00 + +# ATT: {nf} imulq $123456, 291(%r8,%rax,4), %r9 +# INTEL: {nf} imul r9, qword ptr [r8 + 4*rax + 291], 123456 +0x62,0x54,0xfc,0x0c,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00 + +# ATT: {evex} imulb %bl +# INTEL: {evex} imul bl +0x62,0xf4,0x7c,0x08,0xf6,0xeb + +# ATT: {nf} imulb %bl +# INTEL: {nf} imul bl +0x62,0xf4,0x7c,0x0c,0xf6,0xeb + +# ATT: {evex} imulw %dx +# INTEL: {evex} imul dx +0x62,0xf4,0x7d,0x08,0xf7,0xea + +# ATT: {nf} imulw %dx +# INTEL: {nf} imul dx +0x62,0xf4,0x7d,0x0c,0xf7,0xea + +# ATT: {evex} imulw %dx, %dx +# INTEL: {evex} imul dx, dx +0x62,0xf4,0x7d,0x08,0xaf,0xd2 + +# ATT: {nf} imulw %dx, %dx +# INTEL: {nf} imul dx, dx +0x62,0xf4,0x7d,0x0c,0xaf,0xd2 + +# ATT: imulw %dx, %dx, %dx +# INTEL: imul dx, dx, dx +0x62,0xf4,0x6d,0x18,0xaf,0xd2 + +# ATT: {nf} imulw %dx, %dx, %dx +# INTEL: {nf} imul dx, dx, dx +0x62,0xf4,0x6d,0x1c,0xaf,0xd2 + +# ATT: {evex} imull %ecx +# INTEL: {evex} imul ecx +0x62,0xf4,0x7c,0x08,0xf7,0xe9 + +# ATT: {nf} imull %ecx +# INTEL: {nf} imul ecx +0x62,0xf4,0x7c,0x0c,0xf7,0xe9 + +# ATT: {evex} imull %ecx, %ecx +# INTEL: {evex} imul ecx, ecx +0x62,0xf4,0x7c,0x08,0xaf,0xc9 + +# ATT: {nf} imull %ecx, %ecx +# INTEL: {nf} imul ecx, ecx +0x62,0xf4,0x7c,0x0c,0xaf,0xc9 + +# ATT: imull %ecx, %ecx, %ecx +# INTEL: imul ecx, ecx, ecx +0x62,0xf4,0x74,0x18,0xaf,0xc9 + +# ATT: {nf} imull %ecx, %ecx, %ecx +# INTEL: {nf} imul ecx, ecx, ecx +0x62,0xf4,0x74,0x1c,0xaf,0xc9 + +# ATT: {evex} imulq %r9 +# INTEL: {evex} imul r9 +0x62,0xd4,0xfc,0x08,0xf7,0xe9 + +# ATT: {nf} imulq %r9 +# INTEL: {nf} imul r9 +0x62,0xd4,0xfc,0x0c,0xf7,0xe9 + +# ATT: {evex} imulq %r9, %r9 +# INTEL: {evex} imul r9, r9 +0x62,0x54,0xfc,0x08,0xaf,0xc9 + +# ATT: {nf} imulq %r9, %r9 +# INTEL: {nf} imul r9, r9 +0x62,0x54,0xfc,0x0c,0xaf,0xc9 + +# ATT: imulq %r9, %r9, %r9 +# INTEL: imul r9, r9, r9 +0x62,0x54,0xb4,0x18,0xaf,0xc9 + +# ATT: {nf} imulq %r9, %r9, %r9 +# INTEL: {nf} imul r9, r9, r9 +0x62,0x54,0xb4,0x1c,0xaf,0xc9 + +# ATT: {evex} imulb 291(%r8,%rax,4) +# INTEL: {evex} imul byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf6,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imulb 291(%r8,%rax,4) +# INTEL: {nf} imul byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf6,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} imulw 291(%r8,%rax,4) +# INTEL: {evex} imul word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imulw 291(%r8,%rax,4) +# INTEL: {nf} imul word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} imulw 291(%r8,%rax,4), %dx +# INTEL: {evex} imul dx, word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x08,0xaf,0x94,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imulw 291(%r8,%rax,4), %dx +# INTEL: {nf} imul dx, word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x0c,0xaf,0x94,0x80,0x23,0x01,0x00,0x00 + +# ATT: imulw 291(%r8,%rax,4), %dx, %dx +# INTEL: imul dx, dx, word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x6d,0x18,0xaf,0x94,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imulw 291(%r8,%rax,4), %dx, %dx +# INTEL: {nf} imul dx, dx, word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x6d,0x1c,0xaf,0x94,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} imull 291(%r8,%rax,4) +# INTEL: {evex} imul dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imull 291(%r8,%rax,4) +# INTEL: {nf} imul dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} imull 291(%r8,%rax,4), %ecx +# INTEL: {evex} imul ecx, dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imull 291(%r8,%rax,4), %ecx +# INTEL: {nf} imul ecx, dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 + +# ATT: imull 291(%r8,%rax,4), %ecx, %ecx +# INTEL: imul ecx, ecx, dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x74,0x18,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imull 291(%r8,%rax,4), %ecx, %ecx +# INTEL: {nf} imul ecx, ecx, dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x74,0x1c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} imulq 291(%r8,%rax,4) +# INTEL: {evex} imul qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imulq 291(%r8,%rax,4) +# INTEL: {nf} imul qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} imulq 291(%r8,%rax,4), %r9 +# INTEL: {evex} imul r9, qword ptr [r8 + 4*rax + 291] +0x62,0x54,0xfc,0x08,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imulq 291(%r8,%rax,4), %r9 +# INTEL: {nf} imul r9, qword ptr [r8 + 4*rax + 291] +0x62,0x54,0xfc,0x0c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 + +# ATT: imulq 291(%r8,%rax,4), %r9, %r9 +# INTEL: imul r9, r9, qword ptr [r8 + 4*rax + 291] +0x62,0x54,0xb4,0x18,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} imulq 291(%r8,%rax,4), %r9, %r9 +# INTEL: {nf} imul r9, r9, qword ptr [r8 + 4*rax + 291] +0x62,0x54,0xb4,0x1c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/X86/apx/mul.txt b/llvm/test/MC/Disassembler/X86/apx/mul.txt new file mode 100644 index 000000000000..eb5e69e3e2e5 --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/mul.txt @@ -0,0 +1,66 @@ +# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT +# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL + +# ATT: {evex} mulb %bl +# INTEL: {evex} mul bl +0x62,0xf4,0x7c,0x08,0xf6,0xe3 + +# ATT: {nf} mulb %bl +# INTEL: {nf} mul bl +0x62,0xf4,0x7c,0x0c,0xf6,0xe3 + +# ATT: {evex} mulw %dx +# INTEL: {evex} mul dx +0x62,0xf4,0x7d,0x08,0xf7,0xe2 + +# ATT: {nf} mulw %dx +# INTEL: {nf} mul dx +0x62,0xf4,0x7d,0x0c,0xf7,0xe2 + +# ATT: {evex} mull %ecx +# INTEL: {evex} mul ecx +0x62,0xf4,0x7c,0x08,0xf7,0xe1 + +# ATT: {nf} mull %ecx +# INTEL: {nf} mul ecx +0x62,0xf4,0x7c,0x0c,0xf7,0xe1 + +# ATT: {evex} mulq %r9 +# INTEL: {evex} mul r9 +0x62,0xd4,0xfc,0x08,0xf7,0xe1 + +# ATT: {nf} mulq %r9 +# INTEL: {nf} mul r9 +0x62,0xd4,0xfc,0x0c,0xf7,0xe1 + +# ATT: {evex} mulb 291(%r8,%rax,4) +# INTEL: {evex} mul byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf6,0xa4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} mulb 291(%r8,%rax,4) +# INTEL: {nf} mul byte ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf6,0xa4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} mulw 291(%r8,%rax,4) +# INTEL: {evex} mul word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} mulw 291(%r8,%rax,4) +# INTEL: {nf} mul word ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7d,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} mull 291(%r8,%rax,4) +# INTEL: {evex} mul dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} mull 291(%r8,%rax,4) +# INTEL: {nf} mul dword ptr [r8 + 4*rax + 291] +0x62,0xd4,0x7c,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {evex} mulq 291(%r8,%rax,4) +# INTEL: {evex} mul qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00 + +# ATT: {nf} mulq 291(%r8,%rax,4) +# INTEL: {nf} mul qword ptr [r8 + 4*rax + 291] +0x62,0xd4,0xfc,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00 diff --git a/llvm/test/MC/X86/apx/div-att.s b/llvm/test/MC/X86/apx/div-att.s new file mode 100644 index 000000000000..0b78aa5a59d1 --- /dev/null +++ b/llvm/test/MC/X86/apx/div-att.s @@ -0,0 +1,53 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-16: error: +# ERROR-NOT: error: +# CHECK: {evex} divb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xf3] + {evex} divb %bl +# CHECK: {nf} divb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xf3] + {nf} divb %bl +# CHECK: {evex} divw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xf2] + {evex} divw %dx +# CHECK: {nf} divw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xf2] + {nf} divw %dx +# CHECK: {evex} divl %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xf1] + {evex} divl %ecx +# CHECK: {nf} divl %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xf1] + {nf} divl %ecx +# CHECK: {evex} divq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xf1] + {evex} divq %r9 +# CHECK: {nf} divq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xf1] + {nf} divq %r9 +# CHECK: {evex} divb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} divb 291(%r8,%rax,4) +# CHECK: {nf} divb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} divb 291(%r8,%rax,4) +# CHECK: {evex} divw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} divw 291(%r8,%rax,4) +# CHECK: {nf} divw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} divw 291(%r8,%rax,4) +# CHECK: {evex} divl 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} divl 291(%r8,%rax,4) +# CHECK: {nf} divl 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} divl 291(%r8,%rax,4) +# CHECK: {evex} divq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} divq 291(%r8,%rax,4) +# CHECK: {nf} divq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} divq 291(%r8,%rax,4) diff --git a/llvm/test/MC/X86/apx/div-intel.s b/llvm/test/MC/X86/apx/div-intel.s new file mode 100644 index 000000000000..3647b95d5d34 --- /dev/null +++ b/llvm/test/MC/X86/apx/div-intel.s @@ -0,0 +1,50 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s + +# CHECK: {evex} div bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xf3] + {evex} div bl +# CHECK: {nf} div bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xf3] + {nf} div bl +# CHECK: {evex} div dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xf2] + {evex} div dx +# CHECK: {nf} div dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xf2] + {nf} div dx +# CHECK: {evex} div ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xf1] + {evex} div ecx +# CHECK: {nf} div ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xf1] + {nf} div ecx +# CHECK: {evex} div r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xf1] + {evex} div r9 +# CHECK: {nf} div r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xf1] + {nf} div r9 +# CHECK: {evex} div byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} div byte ptr [r8 + 4*rax + 291] +# CHECK: {nf} div byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} div byte ptr [r8 + 4*rax + 291] +# CHECK: {evex} div word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} div word ptr [r8 + 4*rax + 291] +# CHECK: {nf} div word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} div word ptr [r8 + 4*rax + 291] +# CHECK: {evex} div dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} div dword ptr [r8 + 4*rax + 291] +# CHECK: {nf} div dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} div dword ptr [r8 + 4*rax + 291] +# CHECK: {evex} div qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {evex} div qword ptr [r8 + 4*rax + 291] +# CHECK: {nf} div qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xb4,0x80,0x23,0x01,0x00,0x00] + {nf} div qword ptr [r8 + 4*rax + 291] diff --git a/llvm/test/MC/X86/apx/idiv-att.s b/llvm/test/MC/X86/apx/idiv-att.s new file mode 100644 index 000000000000..9b5da5808cd1 --- /dev/null +++ b/llvm/test/MC/X86/apx/idiv-att.s @@ -0,0 +1,53 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-16: error: +# ERROR-NOT: error: +# CHECK: {evex} idivb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xfb] + {evex} idivb %bl +# CHECK: {nf} idivb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xfb] + {nf} idivb %bl +# CHECK: {evex} idivw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xfa] + {evex} idivw %dx +# CHECK: {nf} idivw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xfa] + {nf} idivw %dx +# CHECK: {evex} idivl %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xf9] + {evex} idivl %ecx +# CHECK: {nf} idivl %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xf9] + {nf} idivl %ecx +# CHECK: {evex} idivq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xf9] + {evex} idivq %r9 +# CHECK: {nf} idivq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xf9] + {nf} idivq %r9 +# CHECK: {evex} idivb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idivb 291(%r8,%rax,4) +# CHECK: {nf} idivb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idivb 291(%r8,%rax,4) +# CHECK: {evex} idivw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idivw 291(%r8,%rax,4) +# CHECK: {nf} idivw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idivw 291(%r8,%rax,4) +# CHECK: {evex} idivl 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idivl 291(%r8,%rax,4) +# CHECK: {nf} idivl 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idivl 291(%r8,%rax,4) +# CHECK: {evex} idivq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idivq 291(%r8,%rax,4) +# CHECK: {nf} idivq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idivq 291(%r8,%rax,4) diff --git a/llvm/test/MC/X86/apx/idiv-intel.s b/llvm/test/MC/X86/apx/idiv-intel.s new file mode 100644 index 000000000000..1117df929a16 --- /dev/null +++ b/llvm/test/MC/X86/apx/idiv-intel.s @@ -0,0 +1,50 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s + +# CHECK: {evex} idiv bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xfb] + {evex} idiv bl +# CHECK: {nf} idiv bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xfb] + {nf} idiv bl +# CHECK: {evex} idiv dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xfa] + {evex} idiv dx +# CHECK: {nf} idiv dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xfa] + {nf} idiv dx +# CHECK: {evex} idiv ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xf9] + {evex} idiv ecx +# CHECK: {nf} idiv ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xf9] + {nf} idiv ecx +# CHECK: {evex} idiv r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xf9] + {evex} idiv r9 +# CHECK: {nf} idiv r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xf9] + {nf} idiv r9 +# CHECK: {evex} idiv byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idiv byte ptr [r8 + 4*rax + 291] +# CHECK: {nf} idiv byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idiv byte ptr [r8 + 4*rax + 291] +# CHECK: {evex} idiv word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idiv word ptr [r8 + 4*rax + 291] +# CHECK: {nf} idiv word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idiv word ptr [r8 + 4*rax + 291] +# CHECK: {evex} idiv dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idiv dword ptr [r8 + 4*rax + 291] +# CHECK: {nf} idiv dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idiv dword ptr [r8 + 4*rax + 291] +# CHECK: {evex} idiv qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {evex} idiv qword ptr [r8 + 4*rax + 291] +# CHECK: {nf} idiv qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xbc,0x80,0x23,0x01,0x00,0x00] + {nf} idiv qword ptr [r8 + 4*rax + 291] diff --git a/llvm/test/MC/X86/apx/imul-att.s b/llvm/test/MC/X86/apx/imul-att.s new file mode 100644 index 000000000000..a0a19a423975 --- /dev/null +++ b/llvm/test/MC/X86/apx/imul-att.s @@ -0,0 +1,197 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-64: error: +# ERROR-NOT: error: +# CHECK: {evex} imulw $123, %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0x6b,0xd2,0x7b] + {evex} imulw $123, %dx, %dx +# CHECK: {nf} imulw $123, %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0x6b,0xd2,0x7b] + {nf} imulw $123, %dx, %dx +# CHECK: {evex} imull $123, %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0x6b,0xc9,0x7b] + {evex} imull $123, %ecx, %ecx +# CHECK: {nf} imull $123, %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0x6b,0xc9,0x7b] + {nf} imull $123, %ecx, %ecx +# CHECK: {evex} imulq $123, %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x6b,0xc9,0x7b] + {evex} imulq $123, %r9, %r9 +# CHECK: {nf} imulq $123, %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x6b,0xc9,0x7b] + {nf} imulq $123, %r9, %r9 +# CHECK: {evex} imulw $123, 291(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0x6b,0x94,0x80,0x23,0x01,0x00,0x00,0x7b] + {evex} imulw $123, 291(%r8,%rax,4), %dx +# CHECK: {nf} imulw $123, 291(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0x6b,0x94,0x80,0x23,0x01,0x00,0x00,0x7b] + {nf} imulw $123, 291(%r8,%rax,4), %dx +# CHECK: {evex} imull $123, 291(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {evex} imull $123, 291(%r8,%rax,4), %ecx +# CHECK: {nf} imull $123, 291(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {nf} imull $123, 291(%r8,%rax,4), %ecx +# CHECK: {evex} imulq $123, 291(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {evex} imulq $123, 291(%r8,%rax,4), %r9 +# CHECK: {nf} imulq $123, 291(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {nf} imulq $123, 291(%r8,%rax,4), %r9 +# CHECK: {evex} imulw $1234, %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0x69,0xd2,0xd2,0x04] + {evex} imulw $1234, %dx, %dx +# CHECK: {nf} imulw $1234, %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0x69,0xd2,0xd2,0x04] + {nf} imulw $1234, %dx, %dx +# CHECK: {evex} imulw $1234, 291(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0x69,0x94,0x80,0x23,0x01,0x00,0x00,0xd2,0x04] + {evex} imulw $1234, 291(%r8,%rax,4), %dx +# CHECK: {nf} imulw $1234, 291(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0x69,0x94,0x80,0x23,0x01,0x00,0x00,0xd2,0x04] + {nf} imulw $1234, 291(%r8,%rax,4), %dx +# CHECK: {evex} imull $123456, %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0x69,0xc9,0x40,0xe2,0x01,0x00] + {evex} imull $123456, %ecx, %ecx +# CHECK: {nf} imull $123456, %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0x69,0xc9,0x40,0xe2,0x01,0x00] + {nf} imull $123456, %ecx, %ecx +# CHECK: {evex} imulq $123456, %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x69,0xc9,0x40,0xe2,0x01,0x00] + {evex} imulq $123456, %r9, %r9 +# CHECK: {nf} imulq $123456, %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x69,0xc9,0x40,0xe2,0x01,0x00] + {nf} imulq $123456, %r9, %r9 +# CHECK: {evex} imull $123456, 291(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {evex} imull $123456, 291(%r8,%rax,4), %ecx +# CHECK: {nf} imull $123456, 291(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {nf} imull $123456, 291(%r8,%rax,4), %ecx +# CHECK: {evex} imulq $123456, 291(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {evex} imulq $123456, 291(%r8,%rax,4), %r9 +# CHECK: {nf} imulq $123456, 291(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {nf} imulq $123456, 291(%r8,%rax,4), %r9 +# CHECK: {evex} imulb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xeb] + {evex} imulb %bl +# CHECK: {nf} imulb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xeb] + {nf} imulb %bl +# CHECK: {evex} imulw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xea] + {evex} imulw %dx +# CHECK: {nf} imulw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xea] + {nf} imulw %dx +# CHECK: {evex} imulw %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xaf,0xd2] + {evex} imulw %dx, %dx +# CHECK: {nf} imulw %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xaf,0xd2] + {nf} imulw %dx, %dx +# CHECK: imulw %dx, %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x6d,0x18,0xaf,0xd2] + imulw %dx, %dx, %dx +# CHECK: {nf} imulw %dx, %dx, %dx +# CHECK: encoding: [0x62,0xf4,0x6d,0x1c,0xaf,0xd2] + {nf} imulw %dx, %dx, %dx +# CHECK: {evex} imull %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xe9] + {evex} imull %ecx +# CHECK: {nf} imull %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xe9] + {nf} imull %ecx +# CHECK: {evex} imull %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xaf,0xc9] + {evex} imull %ecx, %ecx +# CHECK: {nf} imull %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xaf,0xc9] + {nf} imull %ecx, %ecx +# CHECK: imull %ecx, %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x74,0x18,0xaf,0xc9] + imull %ecx, %ecx, %ecx +# CHECK: {nf} imull %ecx, %ecx, %ecx +# CHECK: encoding: [0x62,0xf4,0x74,0x1c,0xaf,0xc9] + {nf} imull %ecx, %ecx, %ecx +# CHECK: {evex} imulq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xe9] + {evex} imulq %r9 +# CHECK: {nf} imulq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xe9] + {nf} imulq %r9 +# CHECK: {evex} imulq %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0xaf,0xc9] + {evex} imulq %r9, %r9 +# CHECK: {nf} imulq %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0xaf,0xc9] + {nf} imulq %r9, %r9 +# CHECK: imulq %r9, %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xb4,0x18,0xaf,0xc9] + imulq %r9, %r9, %r9 +# CHECK: {nf} imulq %r9, %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xb4,0x1c,0xaf,0xc9] + {nf} imulq %r9, %r9, %r9 +# CHECK: {evex} imulb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imulb 291(%r8,%rax,4) +# CHECK: {nf} imulb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imulb 291(%r8,%rax,4) +# CHECK: {evex} imulw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imulw 291(%r8,%rax,4) +# CHECK: {nf} imulw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imulw 291(%r8,%rax,4) +# CHECK: {evex} imulw 291(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + {evex} imulw 291(%r8,%rax,4), %dx +# CHECK: {nf} imulw 291(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + {nf} imulw 291(%r8,%rax,4), %dx +# CHECK: imulw 291(%r8,%rax,4), %dx, %dx +# CHECK: encoding: [0x62,0xd4,0x6d,0x18,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + imulw 291(%r8,%rax,4), %dx, %dx +# CHECK: {nf} imulw 291(%r8,%rax,4), %dx, %dx +# CHECK: encoding: [0x62,0xd4,0x6d,0x1c,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + {nf} imulw 291(%r8,%rax,4), %dx, %dx +# CHECK: {evex} imull 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imull 291(%r8,%rax,4) +# CHECK: {nf} imull 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imull 291(%r8,%rax,4) +# CHECK: {evex} imull 291(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {evex} imull 291(%r8,%rax,4), %ecx +# CHECK: {nf} imull 291(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imull 291(%r8,%rax,4), %ecx +# CHECK: imull 291(%r8,%rax,4), %ecx, %ecx +# CHECK: encoding: [0x62,0xd4,0x74,0x18,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + imull 291(%r8,%rax,4), %ecx, %ecx +# CHECK: {nf} imull 291(%r8,%rax,4), %ecx, %ecx +# CHECK: encoding: [0x62,0xd4,0x74,0x1c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imull 291(%r8,%rax,4), %ecx, %ecx +# CHECK: {evex} imulq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imulq 291(%r8,%rax,4) +# CHECK: {nf} imulq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imulq 291(%r8,%rax,4) +# CHECK: {evex} imulq 291(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {evex} imulq 291(%r8,%rax,4), %r9 +# CHECK: {nf} imulq 291(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imulq 291(%r8,%rax,4), %r9 +# CHECK: imulq 291(%r8,%rax,4), %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xb4,0x18,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + imulq 291(%r8,%rax,4), %r9, %r9 +# CHECK: {nf} imulq 291(%r8,%rax,4), %r9, %r9 +# CHECK: encoding: [0x62,0x54,0xb4,0x1c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imulq 291(%r8,%rax,4), %r9, %r9 diff --git a/llvm/test/MC/X86/apx/imul-intel.s b/llvm/test/MC/X86/apx/imul-intel.s new file mode 100644 index 000000000000..b629272afa46 --- /dev/null +++ b/llvm/test/MC/X86/apx/imul-intel.s @@ -0,0 +1,194 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s + +# CHECK: {evex} imul dx, dx, 123 +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0x6b,0xd2,0x7b] + {evex} imul dx, dx, 123 +# CHECK: {nf} imul dx, dx, 123 +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0x6b,0xd2,0x7b] + {nf} imul dx, dx, 123 +# CHECK: {evex} imul ecx, ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0x6b,0xc9,0x7b] + {evex} imul ecx, ecx, 123 +# CHECK: {nf} imul ecx, ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0x6b,0xc9,0x7b] + {nf} imul ecx, ecx, 123 +# CHECK: {evex} imul r9, r9, 123 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x6b,0xc9,0x7b] + {evex} imul r9, r9, 123 +# CHECK: {nf} imul r9, r9, 123 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x6b,0xc9,0x7b] + {nf} imul r9, r9, 123 +# CHECK: {evex} imul dx, word ptr [r8 + 4*rax + 291], 123 +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0x6b,0x94,0x80,0x23,0x01,0x00,0x00,0x7b] + {evex} imul dx, word ptr [r8 + 4*rax + 291], 123 +# CHECK: {nf} imul dx, word ptr [r8 + 4*rax + 291], 123 +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0x6b,0x94,0x80,0x23,0x01,0x00,0x00,0x7b] + {nf} imul dx, word ptr [r8 + 4*rax + 291], 123 +# CHECK: {evex} imul ecx, dword ptr [r8 + 4*rax + 291], 123 +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {evex} imul ecx, dword ptr [r8 + 4*rax + 291], 123 +# CHECK: {nf} imul ecx, dword ptr [r8 + 4*rax + 291], 123 +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {nf} imul ecx, dword ptr [r8 + 4*rax + 291], 123 +# CHECK: {evex} imul r9, qword ptr [r8 + 4*rax + 291], 123 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {evex} imul r9, qword ptr [r8 + 4*rax + 291], 123 +# CHECK: {nf} imul r9, qword ptr [r8 + 4*rax + 291], 123 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x6b,0x8c,0x80,0x23,0x01,0x00,0x00,0x7b] + {nf} imul r9, qword ptr [r8 + 4*rax + 291], 123 +# CHECK: {evex} imul dx, dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0x69,0xd2,0xd2,0x04] + {evex} imul dx, dx, 1234 +# CHECK: {nf} imul dx, dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0x69,0xd2,0xd2,0x04] + {nf} imul dx, dx, 1234 +# CHECK: {evex} imul dx, word ptr [r8 + 4*rax + 291], 1234 +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0x69,0x94,0x80,0x23,0x01,0x00,0x00,0xd2,0x04] + {evex} imul dx, word ptr [r8 + 4*rax + 291], 1234 +# CHECK: {nf} imul dx, word ptr [r8 + 4*rax + 291], 1234 +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0x69,0x94,0x80,0x23,0x01,0x00,0x00,0xd2,0x04] + {nf} imul dx, word ptr [r8 + 4*rax + 291], 1234 +# CHECK: {evex} imul ecx, ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0x69,0xc9,0x40,0xe2,0x01,0x00] + {evex} imul ecx, ecx, 123456 +# CHECK: {nf} imul ecx, ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0x69,0xc9,0x40,0xe2,0x01,0x00] + {nf} imul ecx, ecx, 123456 +# CHECK: {evex} imul r9, r9, 123456 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x69,0xc9,0x40,0xe2,0x01,0x00] + {evex} imul r9, r9, 123456 +# CHECK: {nf} imul r9, r9, 123456 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x69,0xc9,0x40,0xe2,0x01,0x00] + {nf} imul r9, r9, 123456 +# CHECK: {evex} imul ecx, dword ptr [r8 + 4*rax + 291], 123456 +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {evex} imul ecx, dword ptr [r8 + 4*rax + 291], 123456 +# CHECK: {nf} imul ecx, dword ptr [r8 + 4*rax + 291], 123456 +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {nf} imul ecx, dword ptr [r8 + 4*rax + 291], 123456 +# CHECK: {evex} imul r9, qword ptr [r8 + 4*rax + 291], 123456 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {evex} imul r9, qword ptr [r8 + 4*rax + 291], 123456 +# CHECK: {nf} imul r9, qword ptr [r8 + 4*rax + 291], 123456 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0x69,0x8c,0x80,0x23,0x01,0x00,0x00,0x40,0xe2,0x01,0x00] + {nf} imul r9, qword ptr [r8 + 4*rax + 291], 123456 +# CHECK: {evex} imul bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xeb] + {evex} imul bl +# CHECK: {nf} imul bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xeb] + {nf} imul bl +# CHECK: {evex} imul dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xea] + {evex} imul dx +# CHECK: {nf} imul dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xea] + {nf} imul dx +# CHECK: {evex} imul dx, dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xaf,0xd2] + {evex} imul dx, dx +# CHECK: {nf} imul dx, dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xaf,0xd2] + {nf} imul dx, dx +# CHECK: imul dx, dx, dx +# CHECK: encoding: [0x62,0xf4,0x6d,0x18,0xaf,0xd2] + imul dx, dx, dx +# CHECK: {nf} imul dx, dx, dx +# CHECK: encoding: [0x62,0xf4,0x6d,0x1c,0xaf,0xd2] + {nf} imul dx, dx, dx +# CHECK: {evex} imul ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xe9] + {evex} imul ecx +# CHECK: {nf} imul ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xe9] + {nf} imul ecx +# CHECK: {evex} imul ecx, ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xaf,0xc9] + {evex} imul ecx, ecx +# CHECK: {nf} imul ecx, ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xaf,0xc9] + {nf} imul ecx, ecx +# CHECK: imul ecx, ecx, ecx +# CHECK: encoding: [0x62,0xf4,0x74,0x18,0xaf,0xc9] + imul ecx, ecx, ecx +# CHECK: {nf} imul ecx, ecx, ecx +# CHECK: encoding: [0x62,0xf4,0x74,0x1c,0xaf,0xc9] + {nf} imul ecx, ecx, ecx +# CHECK: {evex} imul r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xe9] + {evex} imul r9 +# CHECK: {nf} imul r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xe9] + {nf} imul r9 +# CHECK: {evex} imul r9, r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0xaf,0xc9] + {evex} imul r9, r9 +# CHECK: {nf} imul r9, r9 +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0xaf,0xc9] + {nf} imul r9, r9 +# CHECK: imul r9, r9, r9 +# CHECK: encoding: [0x62,0x54,0xb4,0x18,0xaf,0xc9] + imul r9, r9, r9 +# CHECK: {nf} imul r9, r9, r9 +# CHECK: encoding: [0x62,0x54,0xb4,0x1c,0xaf,0xc9] + {nf} imul r9, r9, r9 +# CHECK: {evex} imul byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imul byte ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imul byte ptr [r8 + 4*rax + 291] +# CHECK: {evex} imul word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imul word ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imul word ptr [r8 + 4*rax + 291] +# CHECK: {evex} imul dx, word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + {evex} imul dx, word ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul dx, word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + {nf} imul dx, word ptr [r8 + 4*rax + 291] +# CHECK: imul dx, dx, word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x6d,0x18,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + imul dx, dx, word ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul dx, dx, word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x6d,0x1c,0xaf,0x94,0x80,0x23,0x01,0x00,0x00] + {nf} imul dx, dx, word ptr [r8 + 4*rax + 291] +# CHECK: {evex} imul dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imul dword ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imul dword ptr [r8 + 4*rax + 291] +# CHECK: {evex} imul ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {evex} imul ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imul ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: imul ecx, ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x74,0x18,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + imul ecx, ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul ecx, ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x74,0x1c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imul ecx, ecx, dword ptr [r8 + 4*rax + 291] +# CHECK: {evex} imul qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {evex} imul qword ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xac,0x80,0x23,0x01,0x00,0x00] + {nf} imul qword ptr [r8 + 4*rax + 291] +# CHECK: {evex} imul r9, qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0x54,0xfc,0x08,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {evex} imul r9, qword ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul r9, qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0x54,0xfc,0x0c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imul r9, qword ptr [r8 + 4*rax + 291] +# CHECK: imul r9, r9, qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0x54,0xb4,0x18,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + imul r9, r9, qword ptr [r8 + 4*rax + 291] +# CHECK: {nf} imul r9, r9, qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0x54,0xb4,0x1c,0xaf,0x8c,0x80,0x23,0x01,0x00,0x00] + {nf} imul r9, r9, qword ptr [r8 + 4*rax + 291] diff --git a/llvm/test/MC/X86/apx/mul-att.s b/llvm/test/MC/X86/apx/mul-att.s new file mode 100644 index 000000000000..d40123bae23b --- /dev/null +++ b/llvm/test/MC/X86/apx/mul-att.s @@ -0,0 +1,53 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-16: error: +# ERROR-NOT: error: +# CHECK: {evex} mulb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xe3] + {evex} mulb %bl +# CHECK: {nf} mulb %bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xe3] + {nf} mulb %bl +# CHECK: {evex} mulw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xe2] + {evex} mulw %dx +# CHECK: {nf} mulw %dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xe2] + {nf} mulw %dx +# CHECK: {evex} mull %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xe1] + {evex} mull %ecx +# CHECK: {nf} mull %ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xe1] + {nf} mull %ecx +# CHECK: {evex} mulq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xe1] + {evex} mulq %r9 +# CHECK: {nf} mulq %r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xe1] + {nf} mulq %r9 +# CHECK: {evex} mulb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mulb 291(%r8,%rax,4) +# CHECK: {nf} mulb 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mulb 291(%r8,%rax,4) +# CHECK: {evex} mulw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mulw 291(%r8,%rax,4) +# CHECK: {nf} mulw 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mulw 291(%r8,%rax,4) +# CHECK: {evex} mull 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mull 291(%r8,%rax,4) +# CHECK: {nf} mull 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mull 291(%r8,%rax,4) +# CHECK: {evex} mulq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mulq 291(%r8,%rax,4) +# CHECK: {nf} mulq 291(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mulq 291(%r8,%rax,4) diff --git a/llvm/test/MC/X86/apx/mul-intel.s b/llvm/test/MC/X86/apx/mul-intel.s new file mode 100644 index 000000000000..11598addfb51 --- /dev/null +++ b/llvm/test/MC/X86/apx/mul-intel.s @@ -0,0 +1,50 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s + +# CHECK: {evex} mul bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf6,0xe3] + {evex} mul bl +# CHECK: {nf} mul bl +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf6,0xe3] + {nf} mul bl +# CHECK: {evex} mul dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x08,0xf7,0xe2] + {evex} mul dx +# CHECK: {nf} mul dx +# CHECK: encoding: [0x62,0xf4,0x7d,0x0c,0xf7,0xe2] + {nf} mul dx +# CHECK: {evex} mul ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x08,0xf7,0xe1] + {evex} mul ecx +# CHECK: {nf} mul ecx +# CHECK: encoding: [0x62,0xf4,0x7c,0x0c,0xf7,0xe1] + {nf} mul ecx +# CHECK: {evex} mul r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xe1] + {evex} mul r9 +# CHECK: {nf} mul r9 +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xe1] + {nf} mul r9 +# CHECK: {evex} mul byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf6,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mul byte ptr [r8 + 4*rax + 291] +# CHECK: {nf} mul byte ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf6,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mul byte ptr [r8 + 4*rax + 291] +# CHECK: {evex} mul word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mul word ptr [r8 + 4*rax + 291] +# CHECK: {nf} mul word ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7d,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mul word ptr [r8 + 4*rax + 291] +# CHECK: {evex} mul dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mul dword ptr [r8 + 4*rax + 291] +# CHECK: {nf} mul dword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0x7c,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mul dword ptr [r8 + 4*rax + 291] +# CHECK: {evex} mul qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x08,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {evex} mul qword ptr [r8 + 4*rax + 291] +# CHECK: {nf} mul qword ptr [r8 + 4*rax + 291] +# CHECK: encoding: [0x62,0xd4,0xfc,0x0c,0xf7,0xa4,0x80,0x23,0x01,0x00,0x00] + {nf} mul qword ptr [r8 + 4*rax + 291] diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc index 4e37285b08ba..02e4ae52cc91 100644 --- a/llvm/test/TableGen/x86-fold-tables.inc +++ b/llvm/test/TableGen/x86-fold-tables.inc @@ -304,18 +304,30 @@ static const X86FoldTableEntry Table0[] = { {X86::CMP8ri8, X86::CMP8mi8, TB_FOLDED_LOAD}, {X86::CMP8rr, X86::CMP8mr, TB_FOLDED_LOAD}, {X86::DIV16r, X86::DIV16m, TB_FOLDED_LOAD}, + {X86::DIV16r_NF, X86::DIV16m_NF, TB_FOLDED_LOAD}, {X86::DIV32r, X86::DIV32m, TB_FOLDED_LOAD}, + {X86::DIV32r_NF, X86::DIV32m_NF, TB_FOLDED_LOAD}, {X86::DIV64r, X86::DIV64m, TB_FOLDED_LOAD}, + {X86::DIV64r_NF, X86::DIV64m_NF, TB_FOLDED_LOAD}, {X86::DIV8r, X86::DIV8m, TB_FOLDED_LOAD}, + {X86::DIV8r_NF, X86::DIV8m_NF, TB_FOLDED_LOAD}, {X86::EXTRACTPSrr, X86::EXTRACTPSmr, TB_FOLDED_STORE}, {X86::IDIV16r, X86::IDIV16m, TB_FOLDED_LOAD}, + {X86::IDIV16r_NF, X86::IDIV16m_NF, TB_FOLDED_LOAD}, {X86::IDIV32r, X86::IDIV32m, TB_FOLDED_LOAD}, + {X86::IDIV32r_NF, X86::IDIV32m_NF, TB_FOLDED_LOAD}, {X86::IDIV64r, X86::IDIV64m, TB_FOLDED_LOAD}, + {X86::IDIV64r_NF, X86::IDIV64m_NF, TB_FOLDED_LOAD}, {X86::IDIV8r, X86::IDIV8m, TB_FOLDED_LOAD}, + {X86::IDIV8r_NF, X86::IDIV8m_NF, TB_FOLDED_LOAD}, {X86::IMUL16r, X86::IMUL16m, TB_FOLDED_LOAD}, + {X86::IMUL16r_NF, X86::IMUL16m_NF, TB_FOLDED_LOAD}, {X86::IMUL32r, X86::IMUL32m, TB_FOLDED_LOAD}, + {X86::IMUL32r_NF, X86::IMUL32m_NF, TB_FOLDED_LOAD}, {X86::IMUL64r, X86::IMUL64m, TB_FOLDED_LOAD}, + {X86::IMUL64r_NF, X86::IMUL64m_NF, TB_FOLDED_LOAD}, {X86::IMUL8r, X86::IMUL8m, TB_FOLDED_LOAD}, + {X86::IMUL8r_NF, X86::IMUL8m_NF, TB_FOLDED_LOAD}, {X86::JMP16r, X86::JMP16m, TB_FOLDED_LOAD}, {X86::JMP16r_NT, X86::JMP16m_NT, TB_FOLDED_LOAD}, {X86::JMP32r, X86::JMP32m, TB_FOLDED_LOAD}, @@ -347,9 +359,13 @@ static const X86FoldTableEntry Table0[] = { {X86::MOVUPDrr, X86::MOVUPDmr, TB_FOLDED_STORE|TB_NO_REVERSE}, {X86::MOVUPSrr, X86::MOVUPSmr, TB_FOLDED_STORE|TB_NO_REVERSE}, {X86::MUL16r, X86::MUL16m, TB_FOLDED_LOAD}, + {X86::MUL16r_NF, X86::MUL16m_NF, TB_FOLDED_LOAD}, {X86::MUL32r, X86::MUL32m, TB_FOLDED_LOAD}, + {X86::MUL32r_NF, X86::MUL32m_NF, TB_FOLDED_LOAD}, {X86::MUL64r, X86::MUL64m, TB_FOLDED_LOAD}, + {X86::MUL64r_NF, X86::MUL64m_NF, TB_FOLDED_LOAD}, {X86::MUL8r, X86::MUL8m, TB_FOLDED_LOAD}, + {X86::MUL8r_NF, X86::MUL8m_NF, TB_FOLDED_LOAD}, {X86::PEXTRDrr, X86::PEXTRDmr, TB_FOLDED_STORE}, {X86::PEXTRQrr, X86::PEXTRQmr, TB_FOLDED_STORE}, {X86::PTWRITE64r, X86::PTWRITE64m, TB_FOLDED_LOAD}, @@ -622,10 +638,16 @@ static const X86FoldTableEntry Table1[] = { {X86::DEC8r_NF_ND, X86::DEC8m_NF_ND, 0}, {X86::IMUL16rri, X86::IMUL16rmi, 0}, {X86::IMUL16rri8, X86::IMUL16rmi8, 0}, + {X86::IMUL16rri8_NF, X86::IMUL16rmi8_NF, 0}, + {X86::IMUL16rri_NF, X86::IMUL16rmi_NF, 0}, {X86::IMUL32rri, X86::IMUL32rmi, 0}, {X86::IMUL32rri8, X86::IMUL32rmi8, 0}, + {X86::IMUL32rri8_NF, X86::IMUL32rmi8_NF, 0}, + {X86::IMUL32rri_NF, X86::IMUL32rmi_NF, 0}, {X86::IMUL64rri32, X86::IMUL64rmi32, 0}, + {X86::IMUL64rri32_NF, X86::IMUL64rmi32_NF, 0}, {X86::IMUL64rri8, X86::IMUL64rmi8, 0}, + {X86::IMUL64rri8_NF, X86::IMUL64rmi8_NF, 0}, {X86::INC16r_ND, X86::INC16m_ND, 0}, {X86::INC16r_NF_ND, X86::INC16m_NF_ND, 0}, {X86::INC32r_ND, X86::INC32m_ND, 0}, @@ -1665,8 +1687,17 @@ static const X86FoldTableEntry Table2[] = { {X86::HSUBPDrr, X86::HSUBPDrm, TB_ALIGN_16}, {X86::HSUBPSrr, X86::HSUBPSrm, TB_ALIGN_16}, {X86::IMUL16rr, X86::IMUL16rm, 0}, + {X86::IMUL16rr_ND, X86::IMUL16rm_ND, 0}, + {X86::IMUL16rr_NF, X86::IMUL16rm_NF, 0}, + {X86::IMUL16rr_NF_ND, X86::IMUL16rm_NF_ND, 0}, {X86::IMUL32rr, X86::IMUL32rm, 0}, + {X86::IMUL32rr_ND, X86::IMUL32rm_ND, 0}, + {X86::IMUL32rr_NF, X86::IMUL32rm_NF, 0}, + {X86::IMUL32rr_NF_ND, X86::IMUL32rm_NF_ND, 0}, {X86::IMUL64rr, X86::IMUL64rm, 0}, + {X86::IMUL64rr_ND, X86::IMUL64rm_ND, 0}, + {X86::IMUL64rr_NF, X86::IMUL64rm_NF, 0}, + {X86::IMUL64rr_NF_ND, X86::IMUL64rm_NF_ND, 0}, {X86::MAXCPDrr, X86::MAXCPDrm, TB_ALIGN_16}, {X86::MAXCPSrr, X86::MAXCPSrm, TB_ALIGN_16}, {X86::MAXCSDrr, X86::MAXCSDrm, 0}, -- GitLab From a72ab9c14d604ae857acad5ed7d6b5be6914c619 Mon Sep 17 00:00:00 2001 From: Hristo Hristov Date: Fri, 5 Jan 2024 11:41:22 +0200 Subject: [PATCH 087/728] [libc++][span] P2821R5: `span.at()` (#74994) - Implements: [P2821R5: span.at()](https://wg21.link/P2821R5) (https://eel.is/c++draft/views.contiguous#views.span) - Cleaned up `span.operator[]` test --------- Co-authored-by: Zingam --- libcxx/docs/FeatureTestMacroTable.rst | 2 + libcxx/docs/ReleaseNotes/18.rst | 1 + libcxx/docs/Status/Cxx2cPapers.csv | 2 +- libcxx/include/span | 18 ++ libcxx/include/version | 2 + .../test/libcxx/transitive_includes/cxx03.csv | 1 + .../test/libcxx/transitive_includes/cxx11.csv | 1 + .../test/libcxx/transitive_includes/cxx14.csv | 1 + .../test/libcxx/transitive_includes/cxx17.csv | 1 + .../test/libcxx/transitive_includes/cxx20.csv | 1 + .../test/libcxx/transitive_includes/cxx23.csv | 1 + .../test/libcxx/transitive_includes/cxx26.csv | 1 + .../views/views.span/span.elem/at.pass.cpp | 188 ++++++++++++++++++ .../views.span/span.elem/op_idx.pass.cpp | 1 - .../span.version.compile.pass.cpp | 28 +++ .../version.version.compile.pass.cpp | 28 +++ .../generate_feature_test_macro_components.py | 5 + 17 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 libcxx/test/std/containers/views/views.span/span.elem/at.pass.cpp diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst index ad12b1090231..8ce5ec9f64ef 100644 --- a/libcxx/docs/FeatureTestMacroTable.rst +++ b/libcxx/docs/FeatureTestMacroTable.rst @@ -436,6 +436,8 @@ Status --------------------------------------------------- ----------------- ``__cpp_lib_smart_ptr_owner_equality`` *unimplemented* --------------------------------------------------- ----------------- + ``__cpp_lib_span_at`` ``202311L`` + --------------------------------------------------- ----------------- ``__cpp_lib_span_initializer_list`` *unimplemented* --------------------------------------------------- ----------------- ``__cpp_lib_sstream_from_string_view`` *unimplemented* diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst index f5cda9aaa5dc..cae2347be5fd 100644 --- a/libcxx/docs/ReleaseNotes/18.rst +++ b/libcxx/docs/ReleaseNotes/18.rst @@ -57,6 +57,7 @@ Implemented Papers - P2871R3 - Remove Deprecated Unicode Conversion Facets from C++26 - P2870R3 - Remove basic_string::reserve() - P2909R4 - Fix formatting of code units as integers (Dude, where’s my ``char``?) +- P2821R5 - span.at() - P0521R0 - Proposed Resolution for CA 14 (shared_ptr use_count/unique) diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index ff83648aa768..fa4a112d1436 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -35,7 +35,7 @@ "`P2909R4 `__","LWG","Fix formatting of code units as integers (Dude, where’s my ``char``?)","Kona November 2023","|Complete|","18.0","|format| |DR|" "`P0952R2 `__","LWG","A new specification for ``std::generate_canonical``","Kona November 2023","","","" "`P2447R6 `__","LWG","``std::span`` over an initializer list","Kona November 2023","","","" -"`P2821R5 `__","LWG","``span.at()``","Kona November 2023","","","" +"`P2821R5 `__","LWG","``span.at()``","Kona November 2023","|Complete|","18.0","" "`P2868R3 `__","LWG","Remove Deprecated ``std::allocator`` Typedef From C++26","Kona November 2023","","","" "`P2870R3 `__","LWG","Remove ``basic_string::reserve()`` From C++26","Kona November 2023","|Complete|","18.0","" "`P2871R3 `__","LWG","Remove Deprecated Unicode Conversion Facets from C++26","Kona November 2023","|Complete|","18.0","" diff --git a/libcxx/include/span b/libcxx/include/span index 7dd53110ac29..007a32597f96 100644 --- a/libcxx/include/span +++ b/libcxx/include/span @@ -92,6 +92,7 @@ public: // [span.elem], span element access constexpr reference operator[](size_type idx) const; + constexpr reference at(size_type idx) const; // since C++26 constexpr reference front() const; constexpr reference back() const; constexpr pointer data() const noexcept; @@ -146,6 +147,7 @@ template #include <__utility/forward.h> #include // for array #include // for byte +#include #include // standard-mandated includes @@ -321,6 +323,14 @@ public: return __data_[__idx]; } +# if _LIBCPP_STD_VER >= 26 + _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const { + if (__index >= size()) + std::__throw_out_of_range("span"); + return __data_[__index]; + } +# endif + _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span::front() on empty span"); return __data_[0]; @@ -469,6 +479,14 @@ public: return __data_[__idx]; } +# if _LIBCPP_STD_VER >= 26 + _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const { + if (__index >= size()) + std::__throw_out_of_range("span"); + return __data_[__index]; + } +# endif + _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept { _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span::front() on empty span"); return __data_[0]; diff --git a/libcxx/include/version b/libcxx/include/version index 768710ef5c84..d3c2791a7d0b 100644 --- a/libcxx/include/version +++ b/libcxx/include/version @@ -201,6 +201,7 @@ __cpp_lib_smart_ptr_for_overwrite 202002L __cpp_lib_smart_ptr_owner_equality 202306L __cpp_lib_source_location 201907L __cpp_lib_span 202002L +__cpp_lib_span_at 202311L __cpp_lib_span_initializer_list 202311L __cpp_lib_spanstream 202106L __cpp_lib_ssize 201902L @@ -505,6 +506,7 @@ __cpp_lib_within_lifetime 202306L // # define __cpp_lib_rcu 202306L // # define __cpp_lib_saturation_arithmetic 202311L // # define __cpp_lib_smart_ptr_owner_equality 202306L +# define __cpp_lib_span_at 202311L // # define __cpp_lib_span_initializer_list 202311L // # define __cpp_lib_sstream_from_string_view 202306L // # define __cpp_lib_submdspan 202306L diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv index 3f0663427176..e709628ce231 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx03.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv @@ -742,6 +742,7 @@ span functional span initializer_list span iterator span limits +span stdexcept span type_traits span version sstream cstddef diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv index 7b443e5a0ec0..d3ea6ed97367 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx11.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv @@ -748,6 +748,7 @@ span functional span initializer_list span iterator span limits +span stdexcept span type_traits span version sstream cstddef diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv index a5b77ec79bb5..3e56017bfb53 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx14.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv @@ -750,6 +750,7 @@ span functional span initializer_list span iterator span limits +span stdexcept span type_traits span version sstream cstddef diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv index a5b77ec79bb5..3e56017bfb53 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx17.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv @@ -750,6 +750,7 @@ span functional span initializer_list span iterator span limits +span stdexcept span type_traits span version sstream cstddef diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv index c8c84867fda4..0c5b9721a225 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx20.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv @@ -755,6 +755,7 @@ span functional span initializer_list span iterator span limits +span stdexcept span type_traits span version sstream cstddef diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv index a4fa50dc014a..0d8c3fa21b2f 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx23.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv @@ -519,6 +519,7 @@ span array span cstddef span initializer_list span limits +span stdexcept span version sstream cstddef sstream istream diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv index a4fa50dc014a..0d8c3fa21b2f 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx26.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv @@ -519,6 +519,7 @@ span array span cstddef span initializer_list span limits +span stdexcept span version sstream cstddef sstream istream diff --git a/libcxx/test/std/containers/views/views.span/span.elem/at.pass.cpp b/libcxx/test/std/containers/views/views.span/span.elem/at.pass.cpp new file mode 100644 index 000000000000..c09571675c0f --- /dev/null +++ b/libcxx/test/std/containers/views/views.span/span.elem/at.pass.cpp @@ -0,0 +1,188 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// constexpr reference at(size_type idx) const; // since C++26 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test_macros.h" + +template +constexpr void testSpanAt(auto&& anySpan, int index, int expectedValue) { + // non-const + { + std::same_as decltype(auto) elem = anySpan.at(index); + assert(elem == expectedValue); + } + + // const + { + std::same_as decltype(auto) elem = std::as_const(anySpan).at(index); + assert(elem == expectedValue); + } +} + +constexpr bool test() { + // With static extent + { + std::array arr{0, 1, 2, 3, 4, 5, 9084}; + std::span arrSpan{arr}; + + assert(std::dynamic_extent != arrSpan.extent); + + using ReferenceT = typename decltype(arrSpan)::reference; + + testSpanAt(arrSpan, 0, 0); + testSpanAt(arrSpan, 1, 1); + testSpanAt(arrSpan, 6, 9084); + } + + // With dynamic extent + { + std::vector vec{0, 1, 2, 3, 4, 5, 9084}; + std::span vecSpan{vec}; + + assert(std::dynamic_extent == vecSpan.extent); + + using ReferenceT = typename decltype(vecSpan)::reference; + + testSpanAt(vecSpan, 0, 0); + testSpanAt(vecSpan, 1, 1); + testSpanAt(vecSpan, 6, 9084); + } + + return true; +} + +void test_exceptions() { +#ifndef TEST_HAS_NO_EXCEPTIONS + using namespace std::string_literals; + + // With static extent + { + std::array arr{0, 1, 2, 3, 4, 5, 9084, std::numeric_limits::max()}; + const std::span arrSpan{arr}; + + try { + using SizeT = typename decltype(arrSpan)::size_type; + std::ignore = arrSpan.at(std::numeric_limits::max()); + assert(false); + } catch ([[maybe_unused]] const std::out_of_range& e) { + // pass + LIBCPP_ASSERT(e.what() == "span"s); + } catch (...) { + assert(false); + } + + try { + std::ignore = arrSpan.at(arr.size()); + assert(false); + } catch ([[maybe_unused]] const std::out_of_range& e) { + // pass + LIBCPP_ASSERT(e.what() == "span"s); + } catch (...) { + assert(false); + } + + try { + std::ignore = arrSpan.at(arr.size() - 1); + // pass + assert(arrSpan.at(arr.size() - 1) == std::numeric_limits::max()); + } catch (...) { + assert(false); + } + } + + { + std::array arr{}; + const std::span arrSpan{arr}; + + try { + std::ignore = arrSpan.at(0); + assert(false); + } catch ([[maybe_unused]] const std::out_of_range& e) { + // pass + LIBCPP_ASSERT(e.what() == "span"s); + } catch (...) { + assert(false); + } + } + + // With dynamic extent + + { + std::vector vec{0, 1, 2, 3, 4, 5, 9084, std::numeric_limits::max()}; + const std::span vecSpan{vec}; + + try { + using SizeT = typename decltype(vecSpan)::size_type; + std::ignore = vecSpan.at(std::numeric_limits::max()); + assert(false); + } catch ([[maybe_unused]] const std::out_of_range& e) { + // pass + LIBCPP_ASSERT(e.what() == "span"s); + } catch (...) { + assert(false); + } + + try { + std::ignore = vecSpan.at(vec.size()); + assert(false); + } catch (const std::out_of_range& e) { + // pass + LIBCPP_ASSERT(e.what() == "span"s); + } catch (...) { + assert(false); + } + + try { + std::ignore = vecSpan.at(vec.size() - 1); + assert(vecSpan.at(vec.size() - 1) == std::numeric_limits::max()); + } catch (...) { + assert(false); + } + } + + { + std::vector vec{}; + const std::span vecSpan{vec}; + + try { + std::ignore = vecSpan.at(0); + assert(false); + } catch ([[maybe_unused]] const std::out_of_range& e) { + // pass + LIBCPP_ASSERT(e.what() == "span"s); + } catch (...) { + assert(false); + } + } +#endif // TEST_HAS_NO_EXCEPTIONS +} + +int main(int, char**) { + test(); + static_assert(test()); + + test_exceptions(); + + return 0; +} diff --git a/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp b/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp index e46fd267ef5c..b7f36c575858 100644 --- a/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp +++ b/libcxx/test/std/containers/views/views.span/span.elem/op_idx.pass.cpp @@ -41,7 +41,6 @@ void testRuntimeSpan(Span sp, std::size_t idx) assert(r1 == r2); } -struct A{}; constexpr int iArr1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; int iArr2[] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19}; diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/span.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/span.version.compile.pass.cpp index 355eb1338d94..dbbbaf4ec7c2 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/span.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/span.version.compile.pass.cpp @@ -17,6 +17,7 @@ /* Constant Value __cpp_lib_span 202002L [C++20] + __cpp_lib_span_at 202311L [C++26] __cpp_lib_span_initializer_list 202311L [C++26] */ @@ -29,6 +30,10 @@ # error "__cpp_lib_span should not be defined before c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -39,6 +44,10 @@ # error "__cpp_lib_span should not be defined before c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -49,6 +58,10 @@ # error "__cpp_lib_span should not be defined before c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -62,6 +75,10 @@ # error "__cpp_lib_span should have the value 202002L in c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -75,6 +92,10 @@ # error "__cpp_lib_span should have the value 202002L in c++23" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -88,6 +109,13 @@ # error "__cpp_lib_span should have the value 202002L in c++26" # endif +# ifndef __cpp_lib_span_at +# error "__cpp_lib_span_at should be defined in c++26" +# endif +# if __cpp_lib_span_at != 202311L +# error "__cpp_lib_span_at should have the value 202311L in c++26" +# endif + # if !defined(_LIBCPP_VERSION) # ifndef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should be defined in c++26" diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp index 650a14b019ed..3b7f2d26feeb 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp @@ -186,6 +186,7 @@ __cpp_lib_smart_ptr_owner_equality 202306L [C++26] __cpp_lib_source_location 201907L [C++20] __cpp_lib_span 202002L [C++20] + __cpp_lib_span_at 202311L [C++26] __cpp_lib_span_initializer_list 202311L [C++26] __cpp_lib_spanstream 202106L [C++23] __cpp_lib_ssize 201902L [C++20] @@ -879,6 +880,10 @@ # error "__cpp_lib_span should not be defined before c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -1716,6 +1721,10 @@ # error "__cpp_lib_span should not be defined before c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -2733,6 +2742,10 @@ # error "__cpp_lib_span should not be defined before c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -4029,6 +4042,10 @@ # error "__cpp_lib_span should have the value 202002L in c++20" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -5541,6 +5558,10 @@ # error "__cpp_lib_span should have the value 202002L in c++23" # endif +# ifdef __cpp_lib_span_at +# error "__cpp_lib_span_at should not be defined before c++26" +# endif + # ifdef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should not be defined before c++26" # endif @@ -7272,6 +7293,13 @@ # error "__cpp_lib_span should have the value 202002L in c++26" # endif +# ifndef __cpp_lib_span_at +# error "__cpp_lib_span_at should be defined in c++26" +# endif +# if __cpp_lib_span_at != 202311L +# error "__cpp_lib_span_at should have the value 202311L in c++26" +# endif + # if !defined(_LIBCPP_VERSION) # ifndef __cpp_lib_span_initializer_list # error "__cpp_lib_span_initializer_list should be defined in c++26" diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py index 2f506f32f565..3ad5170d73ff 100755 --- a/libcxx/utils/generate_feature_test_macro_components.py +++ b/libcxx/utils/generate_feature_test_macro_components.py @@ -1085,6 +1085,11 @@ feature_test_macros = [ }, "headers": ["span"], }, + { + "name": "__cpp_lib_span_at", + "values": {"c++26": 202311}, # P2821R3 span.at() + "headers": ["span"], + }, { "name": "__cpp_lib_span_initializer_list", "values": {"c++26": 202311}, # P2447R6 std::span over an initializer list -- GitLab From 255f95a40377677dd762df5a1aa65bcbb4f75c79 Mon Sep 17 00:00:00 2001 From: Hristo Hristov Date: Fri, 5 Jan 2024 11:42:08 +0200 Subject: [PATCH 088/728] [libc++][streams] P1759R6: Native handles and file streams (#76632) Implements: `P1759R6` https://wg21.link/P1759R6 - https://eel.is/c++draft/filebuf - https://eel.is/c++draft/ifstream - https://eel.is/c++draft/ofstream - https://eel.is/c++draft/fstream --------- Co-authored-by: Zingam --- libcxx/docs/FeatureTestMacroTable.rst | 2 +- libcxx/docs/ReleaseNotes/18.rst | 1 + libcxx/docs/Status/Cxx2cPapers.csv | 2 +- libcxx/include/fstream | 50 ++++++++++ libcxx/include/version | 2 +- libcxx/src/CMakeLists.txt | 1 + libcxx/src/fstream.cpp | 37 +++++++ .../native_handle.assert.pass.cpp | 32 ++++++ .../filebuf.members/native_handle.pass.cpp | 58 +++++++++++ .../fstreams/filebuf/types.pass.cpp | 9 +- .../native_handle.assert.pass.cpp | 32 ++++++ .../fstream.members/native_handle.pass.cpp | 27 ++++++ .../fstreams/fstream/types.pass.cpp | 9 +- .../native_handle.assert.pass.cpp | 32 ++++++ .../ifstream.members/native_handle.pass.cpp | 27 ++++++ .../fstreams/ifstream/types.pass.cpp | 9 +- .../fstreams/native_handle_test_helpers.h | 97 +++++++++++++++++++ .../native_handle.assert.pass.cpp | 32 ++++++ .../ofstream.members/native_handle.pass.cpp | 27 ++++++ .../fstreams/ofstream/types.pass.cpp | 9 +- .../file.streams/fstreams/types.h | 10 ++ .../fstream.version.compile.pass.cpp | 16 +-- .../version.version.compile.pass.cpp | 16 +-- .../generate_feature_test_macro_components.py | 1 - 24 files changed, 508 insertions(+), 30 deletions(-) create mode 100644 libcxx/src/fstream.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp create mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst index 8ce5ec9f64ef..893a3b13ca06 100644 --- a/libcxx/docs/FeatureTestMacroTable.rst +++ b/libcxx/docs/FeatureTestMacroTable.rst @@ -418,7 +418,7 @@ Status --------------------------------------------------- ----------------- ``__cpp_lib_freestanding_variant`` *unimplemented* --------------------------------------------------- ----------------- - ``__cpp_lib_fstream_native_handle`` *unimplemented* + ``__cpp_lib_fstream_native_handle`` ``202306L`` --------------------------------------------------- ----------------- ``__cpp_lib_function_ref`` *unimplemented* --------------------------------------------------- ----------------- diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst index cae2347be5fd..882f53b8d9f8 100644 --- a/libcxx/docs/ReleaseNotes/18.rst +++ b/libcxx/docs/ReleaseNotes/18.rst @@ -59,6 +59,7 @@ Implemented Papers - P2909R4 - Fix formatting of code units as integers (Dude, where’s my ``char``?) - P2821R5 - span.at() - P0521R0 - Proposed Resolution for CA 14 (shared_ptr use_count/unique) +- P1759R6 - Native handles and file streams Improvements and New Features diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index fa4a112d1436..5701717f3976 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -19,7 +19,7 @@ "`P2757R3 `__","LWG","Type-checking format args","Varna June 2023","","","|format|" "`P2637R3 `__","LWG","Member ``visit``","Varna June 2023","","","|format|" "`P2641R4 `__","CWG, LWG","Checking if a ``union`` alternative is active","Varna June 2023","","","" -"`P1759R6 `__","LWG","Native handles and file streams","Varna June 2023","","","" +"`P1759R6 `__","LWG","Native handles and file streams","Varna June 2023","|Complete|","18.0","" "`P2697R1 `__","LWG","Interfacing ``bitset`` with ``string_view``","Varna June 2023","|Complete|","18.0","" "`P1383R2 `__","LWG","More ``constexpr`` for ```` and ````","Varna June 2023","","","" "`P2734R0 `__","LWG","Adding the new SI prefixes","Varna June 2023","|Complete|","17.0","" diff --git a/libcxx/include/fstream b/libcxx/include/fstream index 7a4e15b55d56..cd22b53efaad 100644 --- a/libcxx/include/fstream +++ b/libcxx/include/fstream @@ -73,6 +73,7 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; + using native_handle_type = typename basic_filebuf::native_handle_type; // Since C++26 basic_ifstream(); explicit basic_ifstream(const char* s, ios_base::openmode mode = ios_base::in); @@ -85,6 +86,7 @@ public: void swap(basic_ifstream& rhs); basic_filebuf* rdbuf() const; + native_handle_type native_handle() const noexcept; // Since C++26 bool is_open() const; void open(const char* s, ios_base::openmode mode = ios_base::in); void open(const string& s, ios_base::openmode mode = ios_base::in); @@ -110,6 +112,7 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; + using native_handle_type = typename basic_filebuf::native_handle_type; // Since C++26 basic_ofstream(); explicit basic_ofstream(const char* s, ios_base::openmode mode = ios_base::out); @@ -122,6 +125,8 @@ public: void swap(basic_ofstream& rhs); basic_filebuf* rdbuf() const; + native_handle_type native_handle() const noexcept; // Since C++26 + bool is_open() const; void open(const char* s, ios_base::openmode mode = ios_base::out); void open(const string& s, ios_base::openmode mode = ios_base::out); @@ -148,6 +153,7 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; + using native_handle_type = typename basic_filebuf::native_handle_type; // Since C++26 basic_fstream(); explicit basic_fstream(const char* s, ios_base::openmode mode = ios_base::in|ios_base::out); @@ -160,6 +166,7 @@ public: void swap(basic_fstream& rhs); basic_filebuf* rdbuf() const; + native_handle_type native_handle() const noexcept; // Since C++26 bool is_open() const; void open(const char* s, ios_base::openmode mode = ios_base::in|ios_base::out); void open(const string& s, ios_base::openmode mode = ios_base::in|ios_base::out); @@ -210,6 +217,10 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD +# if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_WIN32API) +_LIBCPP_EXPORTED_FROM_ABI void* __filebuf_windows_native_handle(FILE* __file); +# endif + template class _LIBCPP_TEMPLATE_VIS basic_filebuf : public basic_streambuf<_CharT, _Traits> { public: @@ -219,6 +230,15 @@ public: typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; typedef typename traits_type::state_type state_type; +# if _LIBCPP_STD_VER >= 26 +# if defined(_LIBCPP_WIN32API) + using native_handle_type = void*; // HANDLE +# elif __has_include() + using native_handle_type = int; // POSIX file descriptor +# else +# error "Provide a native file handle!" +# endif +# endif // 27.9.1.2 Constructors/destructor: basic_filebuf(); @@ -245,6 +265,18 @@ public: # endif _LIBCPP_HIDE_FROM_ABI basic_filebuf* __open(int __fd, ios_base::openmode __mode); basic_filebuf* close(); +# if _LIBCPP_STD_VER >= 26 + _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { + _LIBCPP_ASSERT_UNCATEGORIZED(this->is_open(), "File must be opened"); +# if defined(_LIBCPP_WIN32API) + return std::__filebuf_windows_native_handle(__file_); +# elif __has_include() + return fileno(__file_); +# else +# error "Provide a way to determine the file native handle!" +# endif + } +# endif // _LIBCPP_STD_VER >= 26 _LIBCPP_HIDE_FROM_ABI inline static const char* __make_mdstring(ios_base::openmode __mode) _NOEXCEPT; @@ -1024,6 +1056,9 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; +# if _LIBCPP_STD_VER >= 26 + using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type; +# endif _LIBCPP_HIDE_FROM_ABI basic_ifstream(); _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in); @@ -1041,6 +1076,9 @@ public: _LIBCPP_HIDE_FROM_ABI void swap(basic_ifstream& __rhs); _LIBCPP_HIDE_FROM_ABI basic_filebuf* rdbuf() const; +# if _LIBCPP_STD_VER >= 26 + _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); } +# endif _LIBCPP_HIDE_FROM_ABI bool is_open() const; void open(const char* __s, ios_base::openmode __mode = ios_base::in); # ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR @@ -1171,6 +1209,9 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; +# if _LIBCPP_STD_VER >= 26 + using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type; +# endif _LIBCPP_HIDE_FROM_ABI basic_ofstream(); _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const char* __s, ios_base::openmode __mode = ios_base::out); @@ -1190,6 +1231,9 @@ public: _LIBCPP_HIDE_FROM_ABI void swap(basic_ofstream& __rhs); _LIBCPP_HIDE_FROM_ABI basic_filebuf* rdbuf() const; +# if _LIBCPP_STD_VER >= 26 + _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); } +# endif _LIBCPP_HIDE_FROM_ABI bool is_open() const; void open(const char* __s, ios_base::openmode __mode = ios_base::out); # ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR @@ -1321,6 +1365,9 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; +# if _LIBCPP_STD_VER >= 26 + using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type; +# endif _LIBCPP_HIDE_FROM_ABI basic_fstream(); _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const char* __s, @@ -1345,6 +1392,9 @@ public: _LIBCPP_HIDE_FROM_ABI void swap(basic_fstream& __rhs); _LIBCPP_HIDE_FROM_ABI basic_filebuf* rdbuf() const; +# if _LIBCPP_STD_VER >= 26 + _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); } +# endif _LIBCPP_HIDE_FROM_ABI bool is_open() const; _LIBCPP_HIDE_FROM_ABI void open(const char* __s, ios_base::openmode __mode = ios_base::in | ios_base::out); # ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR diff --git a/libcxx/include/version b/libcxx/include/version index d3c2791a7d0b..c96647894dce 100644 --- a/libcxx/include/version +++ b/libcxx/include/version @@ -496,7 +496,7 @@ __cpp_lib_within_lifetime 202306L // # define __cpp_lib_freestanding_optional 202311L // # define __cpp_lib_freestanding_string_view 202311L // # define __cpp_lib_freestanding_variant 202311L -// # define __cpp_lib_fstream_native_handle 202306L +# define __cpp_lib_fstream_native_handle 202306L // # define __cpp_lib_function_ref 202306L // # define __cpp_lib_hazard_pointer 202306L // # define __cpp_lib_linalg 202311L diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt index 329964a00136..96e7c6362d8c 100644 --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -84,6 +84,7 @@ endif() if (LIBCXX_ENABLE_LOCALIZATION) list(APPEND LIBCXX_SOURCES + fstream.cpp include/sso_allocator.h ios.cpp ios.instantiations.cpp diff --git a/libcxx/src/fstream.cpp b/libcxx/src/fstream.cpp new file mode 100644 index 000000000000..55a4442b9c78 --- /dev/null +++ b/libcxx/src/fstream.cpp @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__config> +#include +#include + +#if defined(_LIBCPP_WIN32API) +# define WIN32_LEAN_AND_MEAN +# define NOMINMAX +# include +# include +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if defined(_LIBCPP_WIN32API) + +// Confirm that `HANDLE` is `void*` as implemented in `basic_filebuf` +static_assert(std::same_as); + +_LIBCPP_EXPORTED_FROM_ABI void* __filebuf_windows_native_handle(FILE* __file) noexcept { + // https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/get-osfhandle?view=msvc-170 + intptr_t __handle = _get_osfhandle(fileno(__file)); + if (__handle == -1) + return nullptr; + return reinterpret_cast(__handle); +} + +#endif + +_LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp new file mode 100644 index 000000000000..858f32e72165 --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// REQUIRES: has-unix-headers +// REQUIRES: libcpp-hardening-mode={{extensive|debug}} +// XFAIL: availability-verbose_abort-missing + +// + +// class basic_filebuf; + +// native_handle_type native_handle() const noexcept; + +#include + +#include "../native_handle_test_helpers.h" + +int main(int, char**) { + test_native_handle_assertion>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_assertion>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp new file mode 100644 index 000000000000..22cc63e909c5 --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// class basic_filebuf; + +// native_handle_type native_handle() const noexcept; + +#include +#include +#include +#include + +#include "platform_support.h" +#include "test_macros.h" +#include "../native_handle_test_helpers.h" + +template +void test() { + std::basic_filebuf f; + std::filesystem::path p = get_temp_file_name(); + + // non-const + { + assert(f.open(p, std::ios_base::in) != nullptr); + std::same_as decltype(auto) handle = f.native_handle(); + assert(is_handle_valid(handle)); + f.close(); + assert(!is_handle_valid(handle)); + static_assert(noexcept(f.native_handle())); + } + // const + { + assert(f.open(p, std::ios_base::in) != nullptr); + std::same_as decltype(auto) const_handle = std::as_const(f).native_handle(); + assert(is_handle_valid(const_handle)); + f.close(); + assert(!is_handle_valid(const_handle)); + static_assert(noexcept(std::as_const(f).native_handle())); + } +} + +int main(int, char**) { + test(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp index ad4249f4d809..53a5f78d5eff 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp @@ -23,6 +23,7 @@ #include #include "test_macros.h" +#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -32,6 +33,12 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); +#if TEST_STD_VER >= 26 + test_native_handle_type< std::basic_filebuf>(); +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_type< std::basic_filebuf>(); +# endif +#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp new file mode 100644 index 000000000000..2e6cadbc64e3 --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// REQUIRES: has-unix-headers +// REQUIRES: libcpp-hardening-mode={{extensive|debug}} +// XFAIL: availability-verbose_abort-missing + +// + +// class basic_fstream; + +// native_handle_type native_handle() const noexcept; + +#include + +#include "../native_handle_test_helpers.h" + +int main(int, char**) { + test_native_handle_assertion>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_assertion>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp new file mode 100644 index 000000000000..a7229512385a --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// class basic_fstream; + +// native_handle_type native_handle() const noexcept; + +#include "test_macros.h" +#include "../native_handle_test_helpers.h" + +int main(int, char**) { + test_native_handle>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp index 9274b1854bf8..0e275c53ed18 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp @@ -23,6 +23,7 @@ #include #include "test_macros.h" +#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -32,6 +33,12 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); +#if TEST_STD_VER >= 26 + test_native_handle_type< std::basic_fstream>(); +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_type< std::basic_fstream>(); +# endif +#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp new file mode 100644 index 000000000000..5e55b7303409 --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// REQUIRES: has-unix-headers +// REQUIRES: libcpp-hardening-mode={{extensive|debug}} +// XFAIL: availability-verbose_abort-missing + +// + +// class basic_ifstream; + +// native_handle_type native_handle() const noexcept; + +#include + +#include "../native_handle_test_helpers.h" + +int main(int, char**) { + test_native_handle_assertion>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_assertion>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp new file mode 100644 index 000000000000..c2aff949c8fa --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// class basic_ifstream; + +// native_handle_type native_handle() const noexcept; + +#include "test_macros.h" +#include "../native_handle_test_helpers.h" + +int main(int, char**) { + test_native_handle>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp index 15a044021f64..580e1ed22cba 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp @@ -23,6 +23,7 @@ #include #include "test_macros.h" +#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -32,6 +33,12 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); +#if TEST_STD_VER >= 26 + test_native_handle_type< std::basic_ifstream>(); +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_type< std::basic_ifstream>(); +# endif +#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h b/libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h new file mode 100644 index 000000000000..40bb2fe19487 --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h @@ -0,0 +1,97 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TEST_HELPERS_H +#define TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TEST_HELPERS_H + +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# include +# include +#else +# include +#endif + +#include "platform_support.h" +#include "test_macros.h" +#include "types.h" + +#if TEST_STD_VER >= 26 + +# include "check_assertion.h" + +inline bool is_handle_valid(NativeHandleT handle) { +# if defined(_WIN32) + BY_HANDLE_FILE_INFORMATION fileInformation; + return GetFileInformationByHandle(handle, &fileInformation)); +# elif __has_include() // POSIX + return fcntl(handle, F_GETFL) != -1 || errno != EBADF; +# else +# error "Provide a native file handle!" +# endif +} + +template +inline void test_native_handle() { + static_assert( + std::is_same_v::native_handle_type, typename StreamT::native_handle_type>); + + StreamT f; + std::filesystem::path p = get_temp_file_name(); + + // non-const + { + f.open(p); + std::same_as decltype(auto) handle = f.native_handle(); + assert(is_handle_valid(handle)); + assert(f.rdbuf()->native_handle() == handle); + assert(std::as_const(f).rdbuf()->native_handle() == handle); + f.close(); + assert(!is_handle_valid(handle)); + static_assert(noexcept(f.native_handle())); + } + // const + { + f.open(p); + std::same_as decltype(auto) const_handle = std::as_const(f).native_handle(); + assert(is_handle_valid(const_handle)); + assert(f.rdbuf()->native_handle() == const_handle); + assert(std::as_const(f).rdbuf()->native_handle() == const_handle); + f.close(); + assert(!is_handle_valid(const_handle)); + static_assert(noexcept(std::as_const(f).native_handle())); + } +} + +template +inline void test_native_handle_assertion() { + StreamT f; + + // non-const + TEST_LIBCPP_ASSERT_FAILURE(f.native_handle(), "File must be opened"); + // const + TEST_LIBCPP_ASSERT_FAILURE(std::as_const(f).native_handle(), "File must be opened"); +} + +template +inline void test_native_handle_type() { + static_assert(std::is_trivially_copyable_v); + static_assert(std::semiregular); + static_assert(std::is_same_v); +} + +#endif // #if TEST_STD_VER >= 26 + +#endif // TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TEST_HELPERS_H diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp new file mode 100644 index 000000000000..d42aec1e99b9 --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// REQUIRES: has-unix-headers +// REQUIRES: libcpp-hardening-mode={{extensive|debug}} +// XFAIL: availability-verbose_abort-missing + +// + +// class basic_ofstream; + +// native_handle_type native_handle() const noexcept; + +#include + +#include "../native_handle_test_helpers.h" + +int main(int, char**) { + test_native_handle_assertion>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_assertion>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp new file mode 100644 index 000000000000..88b3a00934cc --- /dev/null +++ b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 + +// + +// class basic_ofstream; + +// native_handle_type native_handle() const noexcept; + +#include "test_macros.h" +#include "../native_handle_test_helpers.h" + +int main(int, char**) { + test_native_handle>(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle>(); +#endif + + return 0; +} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp index 0986d5907bac..242b4610af4a 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp @@ -23,6 +23,7 @@ #include #include "test_macros.h" +#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -32,6 +33,12 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); +#if TEST_STD_VER >= 26 + test_native_handle_type< std::basic_ofstream>(); +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + test_native_handle_type< std::basic_ofstream>(); +# endif +#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/types.h b/libcxx/test/std/input.output/file.streams/fstreams/types.h index b919ba201853..29c8ad2d06c4 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/types.h +++ b/libcxx/test/std/input.output/file.streams/fstreams/types.h @@ -80,4 +80,14 @@ struct LibraryDefaultBuffer { void operator()(std::basic_ifstream&) const {} }; +#if TEST_STD_VER >= 26 +# if defined(_WIN32) +using NativeHandleT = void*; // HANDLE +# elif __has_include() +using NativeHandleT = int; // POSIX file descriptor +# else +# error "Provide a native file handle!" +# endif +#endif + #endif // TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TYPES_H diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp index 981f5420ff7e..eab0313b2c1e 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp @@ -56,17 +56,11 @@ #elif TEST_STD_VER > 23 -# if !defined(_LIBCPP_VERSION) -# ifndef __cpp_lib_fstream_native_handle -# error "__cpp_lib_fstream_native_handle should be defined in c++26" -# endif -# if __cpp_lib_fstream_native_handle != 202306L -# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" -# endif -# else // _LIBCPP_VERSION -# ifdef __cpp_lib_fstream_native_handle -# error "__cpp_lib_fstream_native_handle should not be defined because it is unimplemented in libc++!" -# endif +# ifndef __cpp_lib_fstream_native_handle +# error "__cpp_lib_fstream_native_handle should be defined in c++26" +# endif +# if __cpp_lib_fstream_native_handle != 202306L +# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" # endif #endif // TEST_STD_VER > 23 diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp index 3b7f2d26feeb..d5a0839b30f8 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp @@ -6511,17 +6511,11 @@ # endif # endif -# if !defined(_LIBCPP_VERSION) -# ifndef __cpp_lib_fstream_native_handle -# error "__cpp_lib_fstream_native_handle should be defined in c++26" -# endif -# if __cpp_lib_fstream_native_handle != 202306L -# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" -# endif -# else // _LIBCPP_VERSION -# ifdef __cpp_lib_fstream_native_handle -# error "__cpp_lib_fstream_native_handle should not be defined because it is unimplemented in libc++!" -# endif +# ifndef __cpp_lib_fstream_native_handle +# error "__cpp_lib_fstream_native_handle should be defined in c++26" +# endif +# if __cpp_lib_fstream_native_handle != 202306L +# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" # endif # if !defined(_LIBCPP_VERSION) diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py index 3ad5170d73ff..8ee92909dfa5 100755 --- a/libcxx/utils/generate_feature_test_macro_components.py +++ b/libcxx/utils/generate_feature_test_macro_components.py @@ -570,7 +570,6 @@ feature_test_macros = [ "name": "__cpp_lib_fstream_native_handle", "values": {"c++26": 202306}, # P1759R6 Native handles and file streams "headers": ["fstream"], - "unimplemented": True, }, { "name": "__cpp_lib_function_ref", -- GitLab From a001e9718fd974859f2797a9f9ed7bf87d364e4f Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 5 Jan 2024 10:41:44 +0100 Subject: [PATCH 089/728] [SimplifyLibCalls] Don't try to manually reprocess calls The current code for reprocessing the result of fortified libcall simplifications is not correct, because we might simplify to an argument of the original call, and if that is again a libcall, mistakenly think that this is actually the simplification result. Instead of trying to fix this, simply remove the code entirely, because InstCombine nowadays correctly handles reprocessing of SimplifyLibCall results. Fixes https://github.com/llvm/llvm-project/issues/77064. --- .../lib/Transforms/Utils/SimplifyLibCalls.cpp | 20 +------------------ llvm/test/Transforms/InstCombine/pr77064.ll | 18 +++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/pr77064.ll diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp index 760a626c8b6f..a7cd68e860e4 100644 --- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp @@ -3735,26 +3735,8 @@ Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) { // Also try to simplify calls to fortified library functions. if (Value *SimplifiedFortifiedCI = - FortifiedSimplifier.optimizeCall(CI, Builder)) { - // Try to further simplify the result. - CallInst *SimplifiedCI = dyn_cast(SimplifiedFortifiedCI); - if (SimplifiedCI && SimplifiedCI->getCalledFunction()) { - // Ensure that SimplifiedCI's uses are complete, since some calls have - // their uses analyzed. - replaceAllUsesWith(CI, SimplifiedCI); - - // Set insertion point to SimplifiedCI to guarantee we reach all uses - // we might replace later on. - IRBuilderBase::InsertPointGuard Guard(Builder); - Builder.SetInsertPoint(SimplifiedCI); - if (Value *V = optimizeStringMemoryLibCall(SimplifiedCI, Builder)) { - // If we were able to further simplify, remove the now redundant call. - substituteInParent(SimplifiedCI, V); - return V; - } - } + FortifiedSimplifier.optimizeCall(CI, Builder)) return SimplifiedFortifiedCI; - } // Then check for known library functions. if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) { diff --git a/llvm/test/Transforms/InstCombine/pr77064.ll b/llvm/test/Transforms/InstCombine/pr77064.ll new file mode 100644 index 000000000000..b6afecf83f45 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/pr77064.ll @@ -0,0 +1,18 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=instcombine < %s | FileCheck %s + +define void @main(ptr %ptr) { +; CHECK-LABEL: define void @main( +; CHECK-SAME: ptr [[PTR:%.*]]) { +; CHECK-NEXT: [[OPENDIR:%.*]] = call fastcc ptr @opendir(ptr [[PTR]]) +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 1 dereferenceable(596) [[OPENDIR]], i8 0, i64 596, i1 false) +; CHECK-NEXT: ret void +; + %opendir = call fastcc ptr @opendir(ptr %ptr) + %memset = call ptr @__memset_chk(ptr %opendir, i32 0, i64 596, i64 -1) + ret void +} + +declare ptr @__memset_chk(ptr, i32, i64, i64) + +declare fastcc ptr @opendir(ptr) -- GitLab From 4319e1916dd13f5f7f56536acf01899320b82c6a Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Fri, 5 Jan 2024 10:48:55 +0100 Subject: [PATCH 090/728] [mlir][nvgpu] Introduce Multicast Capability to `nvgpu.tma.async.load` (#76935) This PR improves the functionality of the `nvgpu.tma.async.load` Op by adding support for multicast. While we already had this capability in the lower-level `nvvm.cp.async.bulk.tensor.shared.cluster.global` NVVM Op, this PR lowers mask information to the NVVM operation. --- mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td | 16 +++++++------ .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp | 3 ++- .../NVGPU/TransformOps/NVGPUTransformOps.cpp | 2 +- .../Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir | 23 +++++++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td index 440f7d0380eb..7e139663d74b 100644 --- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td +++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td @@ -642,16 +642,18 @@ def NVGPU_TmaAsyncLoadOp : NVGPU_Op<"tma.async.load", [AttrSizedOperandSegments] The Op uses `$barrier` mbarrier based completion mechanism. }]; - let arguments = (ins Arg]>:$dst, - NVGPU_MBarrierGroup:$barriers, - NVGPU_TensorMapDescriptor:$tensorMapDescriptor, - Variadic:$coordinates, - Index:$mbarId, - Optional:$predicate); + let arguments = (ins Arg]>:$dst, + NVGPU_MBarrierGroup:$barriers, + NVGPU_TensorMapDescriptor:$tensorMapDescriptor, + Variadic:$coordinates, + Index:$mbarId, + Optional:$multicastMask, + Optional:$predicate); let assemblyFormat = [{ $tensorMapDescriptor `[` $coordinates `]` `,` $barriers `[` $mbarId `]` `to` $dst - (`,` `predicate` `=` $predicate^)? + (`multicast_mask` `=` $multicastMask^ )? + (`,` `predicate` `=` $predicate^)? attr-dict `:` type($tensorMapDescriptor) `,` type($barriers) `->` type($dst) }]; diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp index 9cd3a5ce65ce..db84e5cf62a5 100644 --- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp +++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp @@ -990,7 +990,8 @@ struct NVGPUTmaAsyncLoadOpLowering } rewriter.replaceOpWithNewOp( op, dest, adaptor.getTensorMapDescriptor(), coords, barrier, - ValueRange{}, Value{}, Value{}, adaptor.getPredicate()); + ValueRange{}, adaptor.getMulticastMask(), Value{}, + adaptor.getPredicate()); return success(); } }; diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp index b68bed3aa53c..aebdd0a4ee41 100644 --- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp +++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp @@ -980,7 +980,7 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad( Value zero = rewriter.create(loc, 0); Operation *loadOp = rewriter.create( loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero, - Value()); + Value(), Value()); loadOps.push_back(loadOp); auto mixedSizes = memref::getMixedSizes(rewriter, loc, sharedMemref); SmallVector symbols(mixedSizes.size()); diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir index e11449e6f7c4..b8a0f75d1cc8 100644 --- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir +++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir @@ -704,6 +704,29 @@ func.func @async_tma_load_pred(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensor func.return } +func.func @async_tma_load_multicast( + %tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, + %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, + %tensorMap5d: !tensorMap5d, %buffer1d: memref<128xf32,3>, + %buffer2d: memref<32x32xf32,3>, %buffer3d: memref<2x32x32xf32,3>, + %buffer4d: memref<2x2x32x32xf32,3>, %buffer5d: memref<2x2x2x32x32xf32,3>, + %mbarrier: !mbarrier, + %multicastMask: i16) { + %c0 = arith.constant 0 : index + %crd0 = arith.constant 0 : index + %crd1 = arith.constant 0 : index + // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}] + nvgpu.tma.async.load %tensorMap1d[%crd0], %mbarrier[%c0] to %buffer1d multicast_mask = %multicastMask : !tensorMap1d, !mbarrier -> memref<128xf32,3> + // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}] + nvgpu.tma.async.load %tensorMap2d[%crd0, %crd1], %mbarrier[%c0] to %buffer2d multicast_mask = %multicastMask : !tensorMap2d, !mbarrier -> memref<32x32xf32,3> + // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}] + nvgpu.tma.async.load %tensorMap3d[%crd0, %crd1, %crd0], %mbarrier[%c0] to %buffer3d multicast_mask = %multicastMask : !tensorMap3d, !mbarrier -> memref<2x32x32xf32,3> + // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] + nvgpu.tma.async.load %tensorMap4d[%crd0, %crd1, %crd1, %crd0], %mbarrier[%c0] to %buffer4d multicast_mask = %multicastMask : !tensorMap4d, !mbarrier -> memref<2x2x32x32xf32,3> + // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}} box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] + nvgpu.tma.async.load %tensorMap5d[%crd0, %crd1, %crd1, %crd0, %crd0], %mbarrier[%c0] to %buffer5d multicast_mask = %multicastMask : !tensorMap5d, !mbarrier -> memref<2x2x2x32x32xf32,3> + func.return +} func.func @create_tensor_map(%devicePtr2d : memref<64x128xf32>, %devicePtr1d : memref<128xf32>) { %crd0 = arith.constant 64 : index -- GitLab From 2952fb349567fd95c4df020c9092e21972481710 Mon Sep 17 00:00:00 2001 From: Dmitriy Smirnov Date: Fri, 5 Jan 2024 09:51:23 +0000 Subject: [PATCH 091/728] [TOSA] Usage of 32bit integer for 'index to float' in rfft2d (#75098) Lowering of rfft2d to linalg now uses index to i32 cast if an output float is of 32bit and cast to i64 otherwise. --- .../Conversion/TosaToLinalg/TosaToLinalg.cpp | 7 ++- .../TosaToLinalg/tosa-to-linalg.mlir | 48 +++++++++---------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp index 7c35389f1e92..678081837b81 100644 --- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp +++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp @@ -2235,8 +2235,11 @@ struct RFFT2dConverter final : public OpRewritePattern { static Value castIndexToFloat(OpBuilder &builder, Location loc, FloatType type, Value value) { - auto integerVal = - builder.create(loc, builder.getI64Type(), value); + auto integerVal = builder.create( + loc, + type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type() + : builder.getI32Type(), + value); return builder.create(loc, type, integerVal); } diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir index 535316e5a372..3931e454da2e 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir @@ -1629,10 +1629,10 @@ func.func @test_static_rfft2d(%arg0: tensor<5x5x8xf32>) -> (tensor<5x5x5xf32>, t // CHECK: %[[EMPTY_1:.*]] = tensor.empty() : tensor<5x5x5xf32> // CHECK: %[[VAR_3:.*]] = linalg.fill ins(%[[CST_ZERO:.*]]: f32) outs(%[[EMPTY_1:.*]] : tensor<5x5x5xf32>) -> tensor<5x5x5xf32> // CHECK: %[[CST_PI:.*]] = arith.constant 6.28318548 : f32 -// CHECK: %[[VAR_5:.*]] = arith.index_castui %[[CST_5:.*]] : index to i64 -// CHECK: %[[VAR_6:.*]] = arith.uitofp %[[VAR_5:.*]] : i64 to f32 -// CHECK: %[[VAR_7:.*]] = arith.index_castui %[[CST_8:.*]] : index to i64 -// CHECK: %[[VAR_8:.*]] = arith.uitofp %[[VAR_7:.*]] : i64 to f32 +// CHECK: %[[VAR_5:.*]] = arith.index_castui %[[CST_5:.*]] : index to i32 +// CHECK: %[[VAR_6:.*]] = arith.uitofp %[[VAR_5:.*]] : i32 to f32 +// CHECK: %[[VAR_7:.*]] = arith.index_castui %[[CST_8:.*]] : index to i32 +// CHECK: %[[VAR_8:.*]] = arith.uitofp %[[VAR_7:.*]] : i32 to f32 // CHECK: linalg.generic { // CHECK: indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP1]]], // CHECK: iterator_types = ["parallel", "parallel", "parallel", "reduction", "reduction"]} @@ -1640,17 +1640,17 @@ func.func @test_static_rfft2d(%arg0: tensor<5x5x8xf32>) -> (tensor<5x5x5xf32>, t // CHECK: outs(%[[VAR_1]], %[[VAR_3]] : tensor<5x5x5xf32>, tensor<5x5x5xf32>) { // CHECK: ^bb0(%[[IN:.*]]: f32, %[[OUT_0:.*]]: f32, %[[OUT_1:.*]]: f32): // CHECK: %[[INDEX_1:.*]] = linalg.index 1 : index -// CHECK: %[[VAR_10:.*]] = arith.index_castui %[[INDEX_1]] : index to i64 -// CHECK: %[[VAR_11:.*]] = arith.uitofp %[[VAR_10]] : i64 to f32 +// CHECK: %[[VAR_10:.*]] = arith.index_castui %[[INDEX_1]] : index to i32 +// CHECK: %[[VAR_11:.*]] = arith.uitofp %[[VAR_10]] : i32 to f32 // CHECK: %[[INDEX_2:.*]] = linalg.index 2 : index -// CHECK: %[[VAR_13:.*]] = arith.index_castui %[[INDEX_2]] : index to i64 -// CHECK: %[[VAR_14:.*]] = arith.uitofp %[[VAR_13]] : i64 to f32 +// CHECK: %[[VAR_13:.*]] = arith.index_castui %[[INDEX_2]] : index to i32 +// CHECK: %[[VAR_14:.*]] = arith.uitofp %[[VAR_13]] : i32 to f32 // CHECK: %[[INDEX_3:.*]] = linalg.index 3 : index -// CHECK: %[[VAR_16:.*]] = arith.index_castui %[[INDEX_3]] : index to i64 -// CHECK: %[[VAR_17:.*]] = arith.uitofp %[[VAR_16]] : i64 to f32 +// CHECK: %[[VAR_16:.*]] = arith.index_castui %[[INDEX_3]] : index to i32 +// CHECK: %[[VAR_17:.*]] = arith.uitofp %[[VAR_16]] : i32 to f32 // CHECK: %[[INDEX_4:.*]] = linalg.index 4 : index -// CHECK: %[[VAR_19:.*]] = arith.index_castui %[[INDEX_4]] : index to i64 -// CHECK: %[[VAR_20:.*]] = arith.uitofp %[[VAR_19]] : i64 to f32 +// CHECK: %[[VAR_19:.*]] = arith.index_castui %[[INDEX_4]] : index to i32 +// CHECK: %[[VAR_20:.*]] = arith.uitofp %[[VAR_19]] : i32 to f32 // CHECK: %[[VAR_21:.*]] = arith.mulf %[[VAR_17]], %[[VAR_11]] : f32 // CHECK: %[[VAR_22:.*]] = arith.mulf %[[VAR_20]], %[[VAR_14]] : f32 // CHECK: %[[XCOMP:.*]] = arith.divf %[[VAR_21]], %[[VAR_6]] : f32 @@ -1699,10 +1699,10 @@ func.func @test_dynamic_rfft2d(%arg0: tensor) -> (tensor, // CHECK: %[[CST_2:.*]] = arith.constant 2 : index // CHECK: %[[DIM_8:.*]] = tensor.dim %[[ARG_0]], %[[CST_2]] : tensor // CHECK: %[[CST_9:.*]] = arith.constant 6.28318548 : f32 -// CHECK: %[[VAR_6:.*]] = arith.index_castui %[[DIM_6]] : index to i64 -// CHECK: %[[VAR_7:.*]] = arith.uitofp %[[VAR_6]] : i64 to f32 -// CHECK: %[[VAR_8:.*]] = arith.index_castui %[[DIM_8]] : index to i64 -// CHECK: %[[VAR_9:.*]] = arith.uitofp %[[VAR_8]] : i64 to f32 +// CHECK: %[[VAR_6:.*]] = arith.index_castui %[[DIM_6]] : index to i32 +// CHECK: %[[VAR_7:.*]] = arith.uitofp %[[VAR_6]] : i32 to f32 +// CHECK: %[[VAR_8:.*]] = arith.index_castui %[[DIM_8]] : index to i32 +// CHECK: %[[VAR_9:.*]] = arith.uitofp %[[VAR_8]] : i32 to f32 // CHECK: linalg.generic { // CHECK: indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP1]]], // CHECK: iterator_types = ["parallel", "parallel", "parallel", "reduction", "reduction"]} @@ -1710,17 +1710,17 @@ func.func @test_dynamic_rfft2d(%arg0: tensor) -> (tensor, // CHECK: outs(%[[VAR_3]], %[[VAR_5]] : tensor, tensor) { // CHECK: ^bb0(%[[IN:.*]]: f32, %[[OUT_0:.*]]: f32, %[[OUT_1:.*]]: f32): // CHECK: %[[INDEX_1:.*]] = linalg.index 1 : index -// CHECK: %[[VAR_12:.*]] = arith.index_castui %[[INDEX_1]] : index to i64 -// CHECK: %[[VAR_13:.*]] = arith.uitofp %[[VAR_12]] : i64 to f32 +// CHECK: %[[VAR_12:.*]] = arith.index_castui %[[INDEX_1]] : index to i32 +// CHECK: %[[VAR_13:.*]] = arith.uitofp %[[VAR_12]] : i32 to f32 // CHECK: %[[INDEX_2:.*]] = linalg.index 2 : index -// CHECK: %[[VAR_15:.*]] = arith.index_castui %[[INDEX_2]] : index to i64 -// CHECK: %[[VAR_16:.*]] = arith.uitofp %[[VAR_15]] : i64 to f32 +// CHECK: %[[VAR_15:.*]] = arith.index_castui %[[INDEX_2]] : index to i32 +// CHECK: %[[VAR_16:.*]] = arith.uitofp %[[VAR_15]] : i32 to f32 // CHECK: %[[INDEX_3:.*]] = linalg.index 3 : index -// CHECK: %[[VAR_18:.*]] = arith.index_castui %[[INDEX_3]] : index to i64 -// CHECK: %[[VAR_19:.*]] = arith.uitofp %[[VAR_18]] : i64 to f32 +// CHECK: %[[VAR_18:.*]] = arith.index_castui %[[INDEX_3]] : index to i32 +// CHECK: %[[VAR_19:.*]] = arith.uitofp %[[VAR_18]] : i32 to f32 // CHECK: %[[INDEX_4:.*]] = linalg.index 4 : index -// CHECK: %[[VAR_21:.*]] = arith.index_castui %[[INDEX_4]] : index to i64 -// CHECK: %[[VAR_22:.*]] = arith.uitofp %[[VAR_21]] : i64 to f32 +// CHECK: %[[VAR_21:.*]] = arith.index_castui %[[INDEX_4]] : index to i32 +// CHECK: %[[VAR_22:.*]] = arith.uitofp %[[VAR_21]] : i32 to f32 // CHECK: %[[VAR_23:.*]] = arith.mulf %[[VAR_19]], %[[VAR_13]] : f32 // CHECK: %[[VAR_24:.*]] = arith.mulf %[[VAR_22]], %[[VAR_16]] : f32 // CHECK: %[[XCOMP:.*]] = arith.divf %[[VAR_23]], %[[VAR_7]] : f32 -- GitLab From 0eefcaf96d2900a4f3009026a3673ed3b7793fcc Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Fri, 5 Jan 2024 09:55:50 +0000 Subject: [PATCH 092/728] [Clang][SME] Add IsStreamingOrSVE2p1 (#76975) This patch adds IsStreamingOrSVE2p1 to the applicable builtins and a warning for when those builtins are not used in a streaming or sve2p1 function. --- clang/include/clang/Basic/arm_sve.td | 115 +++++++-------- clang/include/clang/Basic/arm_sve_sme_incl.td | 1 + clang/lib/Basic/Targets/AArch64.h | 1 + clang/lib/Sema/SemaChecking.cpp | 17 ++- .../aarch64-sve2-intrinsics/acle_sve2_revd.c | 4 +- .../acle_sve2p1_bfmlsl.c | 8 +- .../acle_sve2p1_cntp.c | 27 ++-- .../acle_sve2p1_fclamp.c | 14 +- .../acle_sve2p1_ld1.c | 2 + .../acle_sve2p1_pext.c | 5 +- .../acle_sve2p1_pfalse.c | 14 +- .../acle_sve2p1_psel.c | 26 +++- .../acle_sve2p1_sclamp.c | 20 ++- .../acle_sve2p1_stnt1.c | 2 - .../acle_sve2p1_uclamp.c | 20 ++- .../acle_sve2p1_while_pn.c | 137 +++++++++--------- .../aarch64-sme2-intrinsics/acle_sme2_imm.cpp | 2 +- .../Sema/aarch64-sme2-sve2p1-diagnostics.c | 37 +++++ clang/utils/TableGen/SveEmitter.cpp | 3 + 19 files changed, 285 insertions(+), 170 deletions(-) create mode 100644 clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 91f62c4c7633..7f80fb0386cc 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1976,39 +1976,37 @@ def SVFMINQV: SInst<"svminqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminq } let TargetGuard = "sve2p1|sme2" in { -//FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available -def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>; -def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>; +def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_3>]>; +def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_1>]>; -def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; -def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; +def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>; } multiclass MultiVecLoad { - // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) - def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - - def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; } let TargetGuard = "sve2p1|sme2" in { @@ -2017,24 +2015,23 @@ let TargetGuard = "sve2p1|sme2" in { } multiclass MultiVecStore { - // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming) - def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - - def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; - def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; - def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + + def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; + def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; + def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; } let TargetGuard = "sve2p1|sme2" in { @@ -2051,21 +2048,20 @@ def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "Ui", MergeNone def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "f", MergeNone, "aarch64_sve_fdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>; } -let TargetGuard = "sve2p1|sme" in { -def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; -def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; +let TargetGuard = "sve2p1|sme2" in { +def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [IsStreamingOrSVE2p1], []>; +def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [IsStreamingOrSVE2p1], []>; defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">; } let TargetGuard = "sve2p1|sme2" in { - //FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available - def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingCompatible], []>; + def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], []>; - def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingCompatible]>; + def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingOrSVE2p1]>; - def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingCompatible], []>; - def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<1, ImmCheck2_4_Mul2>]>; + def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingOrSVE2p1], []>; + def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck2_4_Mul2>]>; } let TargetGuard = "(sve2|sme2),b16b16" in { @@ -2326,10 +2322,9 @@ let TargetGuard = "sme2" in { let TargetGuard = "sve2p1|sme2" in { // == BFloat16 multiply-subtract == -// FIXME: Make all of these IsStreamingOrSVE2p1 once that is added - def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingCompatible], []>; - def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingCompatible], []>; + def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingOrSVE2p1], []>; + def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingOrSVE2p1], []>; - def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; - def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>; + def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>; } diff --git a/clang/include/clang/Basic/arm_sve_sme_incl.td b/clang/include/clang/Basic/arm_sve_sme_incl.td index 0dba8493bad2..ad29864440c9 100644 --- a/clang/include/clang/Basic/arm_sve_sme_incl.td +++ b/clang/include/clang/Basic/arm_sve_sme_incl.td @@ -227,6 +227,7 @@ def IsPreservesZA : FlagType<0x10000000000>; def IsReadZA : FlagType<0x20000000000>; def IsWriteZA : FlagType<0x40000000000>; def IsReductionQV : FlagType<0x80000000000>; +def IsStreamingOrSVE2p1 : FlagType<0x80000000000>; // Use for intrinsics that are common between sme/sme2 and sve2p1. // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h class ImmCheckType { diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h index f0e0782e7abe..e38fa5af5659 100644 --- a/clang/lib/Basic/Targets/AArch64.h +++ b/clang/lib/Basic/Targets/AArch64.h @@ -50,6 +50,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo { bool HasMatMul = false; bool HasBFloat16 = false; bool HasSVE2 = false; + bool HasSVE2p1 = false; bool HasSVE2AES = false; bool HasSVE2SHA3 = false; bool HasSVE2SM4 = false; diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 3168d38dd66c..f13164dc0638 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -2998,7 +2998,12 @@ static QualType getNeonEltType(NeonTypeFlags Flags, ASTContext &Context, llvm_unreachable("Invalid NeonTypeFlag!"); } -enum ArmStreamingType { ArmNonStreaming, ArmStreaming, ArmStreamingCompatible }; +enum ArmStreamingType { + ArmNonStreaming, + ArmStreaming, + ArmStreamingCompatible, + ArmStreamingOrSVE2p1 +}; bool Sema::ParseSVEImmChecks( CallExpr *TheCall, SmallVector, 3> &ImmChecks) { @@ -3156,6 +3161,16 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall, const FunctionDecl *FD, ArmStreamingType BuiltinType) { ArmStreamingType FnType = getArmStreamingFnType(FD); + if (BuiltinType == ArmStreamingOrSVE2p1) { + // Check intrinsics that are available in [sve2p1 or sme/sme2]. + llvm::StringMap CallerFeatureMap; + S.Context.getFunctionFeatureMap(CallerFeatureMap, FD); + if (Builtin::evaluateRequiredTargetFeatures("sve2p1", CallerFeatureMap)) + BuiltinType = ArmStreamingCompatible; + else + BuiltinType = ArmStreaming; + } + if (FnType == ArmStreaming && BuiltinType == ArmNonStreaming) { S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin) << TheCall->getSourceRange() << "streaming"; diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c index 74a90583a173..d82d69442b8f 100644 --- a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c +++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c @@ -1,7 +1,7 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ -// RUN: -target-feature +sme -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ @@ -9,7 +9,7 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +bf16 -S -disable-O0-optnone -Werror -o /dev/null %s #include #ifdef SVE_OVERLOADED_FORMS diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c index c1d14e16ad17..22d951c069bc 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c @@ -2,20 +2,20 @@ // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s // RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -target-feature -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include #ifndef TEST_SME2 #define ATTR #else -#define ATTR __arm_streaming_compatible +#define ATTR __arm_streaming #endif #ifdef SVE_OVERLOADED_FORMS diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c index 56b1d9926221..9bf55eaa6a08 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c @@ -3,10 +3,19 @@ // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svcntp_c8_vlx2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2) @@ -17,7 +26,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c8_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c8_vlx2(svcount_t pnn) ATTR { return svcntp_c8(pnn, 2); } @@ -31,7 +40,7 @@ uint64_t test_svcntp_c8_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c8_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c8_vlx4(svcount_t pnn) ATTR { return svcntp_c8(pnn, 4); } @@ -45,7 +54,7 @@ uint64_t test_svcntp_c8_vlx4(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c16_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c16_vlx2(svcount_t pnn) ATTR { return svcntp_c16(pnn, 2); } @@ -59,7 +68,7 @@ uint64_t test_svcntp_c16_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c16_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c16_vlx4(svcount_t pnn) ATTR { return svcntp_c16(pnn, 4); } @@ -73,7 +82,7 @@ uint64_t test_svcntp_c16_vlx4(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c32_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c32_vlx2(svcount_t pnn) ATTR { return svcntp_c32(pnn, 2); } @@ -87,7 +96,7 @@ uint64_t test_svcntp_c32_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c32_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c32_vlx4(svcount_t pnn) ATTR { return svcntp_c32(pnn, 4); } @@ -101,7 +110,7 @@ uint64_t test_svcntp_c32_vlx4(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 2) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c64_vlx2(svcount_t pnn) { +uint64_t test_svcntp_c64_vlx2(svcount_t pnn) ATTR { return svcntp_c64(pnn, 2); } @@ -115,6 +124,6 @@ uint64_t test_svcntp_c64_vlx2(svcount_t pnn) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 4) // CPP-CHECK-NEXT: ret i64 [[TMP0]] // -uint64_t test_svcntp_c64_vlx4(svcount_t pnn) { +uint64_t test_svcntp_c64_vlx4(svcount_t pnn) ATTR { return svcntp_c64(pnn, 4); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c index 5d8c5b7b8a18..7687257701a6 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fclamp.c @@ -11,10 +11,16 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve \ -// RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -32,7 +38,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv8f16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) { +svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _f16, , )(op1, op2, op3); } @@ -46,7 +52,7 @@ svfloat16_t test_svclamp_f16(svfloat16_t op1, svfloat16_t op2, svfloat16_t op3) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv4f32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) { +svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _f32, , )(op1, op2, op3); } @@ -60,7 +66,7 @@ svfloat32_t test_svclamp_f32(svfloat32_t op1, svfloat32_t op2, svfloat32_t op3) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.fclamp.nxv2f64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svfloat64_t test_svclamp_f64(svfloat64_t op1, svfloat64_t op2, svfloat64_t op3) { +svfloat64_t test_svclamp_f64(svfloat64_t op1, svfloat64_t op2, svfloat64_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _f64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c index 6f1231e776aa..7657165d8b3f 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c @@ -1,6 +1,8 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -DTEST_SME2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wno-unknown-attributes -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c index a3206029019c..8f08b32618b0 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c @@ -1,8 +1,11 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -DTEST_SME2 -target-feature +sve -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c index 19993e541812..afdb038fb931 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c @@ -1,14 +1,20 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svpfalse_c( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( zeroinitializer) @@ -19,7 +25,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( zeroinitializer) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svpfalse_c(void) __arm_streaming_compatible +svcount_t test_svpfalse_c(void) ATTR { return svpfalse_c(); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c index 73b7b0347dd9..de3f6a9a57bf 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c @@ -10,9 +10,19 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu \ +// RUN: -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming_compatible +#endif + // CHECK-LABEL: @test_svpsel_lane_b8( // CHECK-NEXT: entry: // CHECK-NEXT: [[ADD:%.*]] = add i32 [[IDX:%.*]], 15 @@ -25,7 +35,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.psel.nxv16i1( [[P1:%.*]], [[P2:%.*]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b8(p1, p2, idx + 15); } @@ -43,7 +53,7 @@ svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) __arm_strea // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv8i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b16(p1, p2, idx + 7); } @@ -61,7 +71,7 @@ svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) __arm_stre // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv4i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b32(p1, p2, idx + 3); } @@ -79,7 +89,7 @@ svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) __arm_stre // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv2i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_b64(p1, p2, idx + 1); } @@ -99,7 +109,7 @@ svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) __arm_stre // CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP1]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP2]] // -svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c8(p1, p2, idx + 15); } @@ -121,7 +131,7 @@ svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) __arm_str // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c16(p1, p2, idx + 7); } @@ -143,7 +153,7 @@ svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) __arm_st // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c32(p1, p2, idx + 3); } @@ -165,6 +175,6 @@ svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) __arm_st // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { +svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) ATTR { return svpsel_lane_c64(p1, p2, idx + 1); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c index 8c63a7455c79..04869fd550ec 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_sclamp.c @@ -10,9 +10,21 @@ // RUN: -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -30,7 +42,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv16i8( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) { +svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s8, , )(op1, op2, op3); } @@ -44,7 +56,7 @@ svint8_t test_svclamp_s8(svint8_t op1, svint8_t op2, svint8_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv8i16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) { +svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s16, , )(op1, op2, op3); } @@ -58,7 +70,7 @@ svint16_t test_svclamp_s16(svint16_t op1, svint16_t op2, svint16_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv4i32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) { +svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s32, , )(op1, op2, op3); } @@ -72,7 +84,7 @@ svint32_t test_svclamp_s32(svint32_t op1, svint32_t op2, svint32_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.sclamp.nxv2i64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svint64_t test_svclamp_s64(svint64_t op1, svint64_t op2, svint64_t op3) { +svint64_t test_svclamp_s64(svint64_t op1, svint64_t op2, svint64_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _s64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c index 0d8696a7634a..b1ca27b7b68a 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c @@ -505,11 +505,9 @@ void test_svstnt1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v) ATTR return SVE_ACLE_FUNC(svstnt1,_f64_x4,,)(pn, base, v); } - // == VNUM variants == - // CHECK-LABEL: @test_svstnt1_vnum_u8_x2( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i8.nxv32i8( [[V:%.*]], i64 0) diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c index b87898624887..37bfd4265a43 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_uclamp.c @@ -10,9 +10,21 @@ // RUN: -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 \ // RUN: -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sme2 \ +// RUN: -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + #ifdef SVE_OVERLOADED_FORMS // A simple used,unused... macro, long enough to represent any SVE builtin. #define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 @@ -30,7 +42,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv16i8( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) { +svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u8, , )(op1, op2, op3); } @@ -44,7 +56,7 @@ svuint8_t test_svclamp_u8(svuint8_t op1, svuint8_t op2, svuint8_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv8i16( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) { +svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u16, , )(op1, op2, op3); } @@ -58,7 +70,7 @@ svuint16_t test_svclamp_u16(svuint16_t op1, svuint16_t op2, svuint16_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv4i32( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) { +svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u32, , )(op1, op2, op3); } @@ -72,7 +84,7 @@ svuint32_t test_svclamp_u32(svuint32_t op1, svuint32_t op2, svuint32_t op3) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.uclamp.nxv2i64( [[OP1:%.*]], [[OP2:%.*]], [[OP3:%.*]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svuint64_t test_svclamp_u64(svuint64_t op1, svuint64_t op2, svuint64_t op3) { +svuint64_t test_svclamp_u64(svuint64_t op1, svuint64_t op2, svuint64_t op3) ATTR { return SVE_ACLE_FUNC(svclamp, _u64, , )(op1, op2, op3); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c index 143a43b4a921..11ebec9e7cbf 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c @@ -1,10 +1,10 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -DTEST_SME2 -disable-O0-optnone -Werror -Wall -o /dev/null %s // REQUIRES: aarch64-registered-target @@ -16,6 +16,11 @@ #define SVE_ACLE_FUNC(A1, A2) A1##A2 #endif +#ifdef TEST_SME2 +#define ATTR __arm_streaming +#else +#define ATTR +#endif // WHILEGE @@ -29,7 +34,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 2); } @@ -44,7 +49,7 @@ svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_s64)(op1, op2, 4); } @@ -59,7 +64,7 @@ svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 2); } @@ -74,7 +79,7 @@ svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_s64)(op1, op2, 4); } @@ -89,7 +94,7 @@ svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 2); } @@ -104,7 +109,7 @@ svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_s64)(op1, op2, 4); } @@ -119,7 +124,7 @@ svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 2); } @@ -134,7 +139,7 @@ svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_s64)(op1, op2, 4); } @@ -152,7 +157,7 @@ svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 2); } @@ -167,7 +172,7 @@ svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_s64)(op1, op2, 4); } @@ -182,7 +187,7 @@ svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 2); } @@ -197,7 +202,7 @@ svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_s64)(op1, op2, 4); } @@ -212,7 +217,7 @@ svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 2); } @@ -227,7 +232,7 @@ svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_s64)(op1, op2, 4); } @@ -242,7 +247,7 @@ svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 2); } @@ -257,7 +262,7 @@ svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_s64)(op1, op2, 4); } @@ -275,7 +280,7 @@ svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 2); } @@ -290,7 +295,7 @@ svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c8,_u64)(op1, op2, 4); } @@ -305,7 +310,7 @@ svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 2); } @@ -320,7 +325,7 @@ svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c16,_u64)(op1, op2, 4); } @@ -335,7 +340,7 @@ svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 2); } @@ -350,7 +355,7 @@ svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c32,_u64)(op1, op2, 4); } @@ -365,7 +370,7 @@ svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 2); } @@ -380,7 +385,7 @@ svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilegt_c64,_u64)(op1, op2, 4); } @@ -398,7 +403,7 @@ svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 2); } @@ -413,7 +418,7 @@ svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c8,_u64)(op1, op2, 4); } @@ -428,7 +433,7 @@ svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 2); } @@ -443,7 +448,7 @@ svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c16,_u64)(op1, op2, 4); } @@ -458,7 +463,7 @@ svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 2); } @@ -473,7 +478,7 @@ svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c32,_u64)(op1, op2, 4); } @@ -488,7 +493,7 @@ svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 2); } @@ -503,7 +508,7 @@ svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilege_c64,_u64)(op1, op2, 4); } @@ -521,7 +526,7 @@ svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 2); } @@ -536,7 +541,7 @@ svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_s64)(op1, op2, 4); } @@ -551,7 +556,7 @@ svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 2); } @@ -566,7 +571,7 @@ svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_s64)(op1, op2, 4); } @@ -581,7 +586,7 @@ svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 2); } @@ -596,7 +601,7 @@ svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_s64)(op1, op2, 4); } @@ -611,7 +616,7 @@ svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 2); } @@ -626,7 +631,7 @@ svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_s64)(op1, op2, 4); } @@ -644,7 +649,7 @@ svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 2); } @@ -659,7 +664,7 @@ svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_u64)(op1, op2, 4); } @@ -674,7 +679,7 @@ svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 2); } @@ -689,7 +694,7 @@ svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_u64)(op1, op2, 4); } @@ -704,7 +709,7 @@ svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 2); } @@ -719,7 +724,7 @@ svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_u64)(op1, op2, 4); } @@ -734,7 +739,7 @@ svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 2); } @@ -749,7 +754,7 @@ svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_u64)(op1, op2, 4); } @@ -767,7 +772,7 @@ svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 2); } @@ -782,7 +787,7 @@ svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c8,_u64)(op1, op2, 4); } @@ -797,7 +802,7 @@ svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 2); } @@ -812,7 +817,7 @@ svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c16,_u64)(op1, op2, 4); } @@ -827,7 +832,7 @@ svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 2); } @@ -842,7 +847,7 @@ svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c32,_u64)(op1, op2, 4); } @@ -857,7 +862,7 @@ svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 2); } @@ -872,7 +877,7 @@ svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) +svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilele_c64,_u64)(op1, op2, 4); } @@ -890,7 +895,7 @@ svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 2); } @@ -905,7 +910,7 @@ svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c8,_s64)(op1, op2, 4); } @@ -920,7 +925,7 @@ svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 2); } @@ -935,7 +940,7 @@ svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c16,_s64)(op1, op2, 4); } @@ -950,7 +955,7 @@ svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 2); } @@ -965,7 +970,7 @@ svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c32,_s64)(op1, op2, 4); } @@ -980,7 +985,7 @@ svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 2); } @@ -995,7 +1000,7 @@ svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2) // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // -svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2) +svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2) ATTR { return SVE_ACLE_FUNC(svwhilelt_c64,_s64)(op1, op2, 4); } diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp index 6a6370bf99b1..5118f743174c 100644 --- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp +++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp @@ -237,7 +237,7 @@ void test_svluti4_lane_zt_x2(svuint8_t zn_u8) __arm_streaming __arm_shared_za __ svluti4_lane_zt_f32_x2(0, zn_u8, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}} } -void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming_compatible { +void test_bfmlslb_bad_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm) __arm_streaming { svbfmlslb_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} svbfmlslt_lane_f32(zda, zn, zm, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} } diff --git a/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c b/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c new file mode 100644 index 000000000000..4debc14190aa --- /dev/null +++ b/clang/test/Sema/aarch64-sme2-sve2p1-diagnostics.c @@ -0,0 +1,37 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -fsyntax-only -verify %s + +// REQUIRES: aarch64-registered-target +#include "arm_sve.h" + +//svldnt1: + +__attribute__((target("+sme2"))) +svuint8x2_t sme2_or_sve2p1_intrinsic_test_sme2_invalid(svcount_t png, const uint8_t *rn) { + // expected-warning@+1 {{builtin call has undefined behaviour when called from a non-streaming function}} + return svldnt1_u8_x2(png, rn); +} + +__attribute__((target("+sme2"))) +svint16x4_t sme2_or_sve2p1_intrinsic_test_sme2(svcount_t png, const int16_t *rn) __arm_streaming { + // expected-no-warning + return svldnt1_s16_x4(png, rn); +} + +__attribute__((target("+sve2p1"))) +svuint32x2_t sme2_or_sve2p1_intrinsic_test_sve2p1(svcount_t png, const uint32_t *rn) { + // expected-no-warning + return svldnt1_u32_x2(png, rn); +} + +__attribute__((target("+sme2,+sve2p1"))) +svint64x4_t sme2_or_sve2p1_intrinsic_test_both_arm_streaming(svcount_t png, const int64_t *rn) __arm_streaming { + // expected-no-warning + return svldnt1_s64_x4(png, rn); +} + +__attribute__((target("+sme2,+sve2p1"))) +svint64x4_t sme2_or_sve2p1_intrinsic_test_both_no_arm_streaming(svcount_t png, const int64_t *rn) { + // expected-no-warning + return svldnt1_s64_x4(png, rn); +} diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp index 6c302da106a2..5de2223e71b0 100644 --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -1773,11 +1773,14 @@ void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) { llvm::StringMap> StreamingMap; uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming"); + uint64_t IsStreamingOrSVE2p1Flag = getEnumValueForFlag("IsStreamingOrSVE2p1"); uint64_t IsStreamingCompatibleFlag = getEnumValueForFlag("IsStreamingCompatible"); for (auto &Def : Defs) { if (Def->isFlagSet(IsStreamingFlag)) StreamingMap["ArmStreaming"].insert(Def->getMangledName()); + else if (Def->isFlagSet(IsStreamingOrSVE2p1Flag)) + StreamingMap["ArmStreamingOrSVE2p1"].insert(Def->getMangledName()); else if (Def->isFlagSet(IsStreamingCompatibleFlag)) StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName()); else -- GitLab From c458f928fad7bbcf08ab1da9949eb2969fc9f89c Mon Sep 17 00:00:00 2001 From: Duo Wang Date: Fri, 5 Jan 2024 01:59:26 -0800 Subject: [PATCH 093/728] [NFC][clang][test][asan] Make `instantiation-depth-default.cpp` a valid test case under `asan` and `ubsan` configs (#75254) Clang test `instantiation-depth-default.cpp` fails on Windows when built with `ubsan` due to extra warnings printed by the compiler: ```console File instantiation-depth-default.cpp Line 11: stack nearly exhausted; compilation time may suffer, and crashes due to stack overflow are likely ``` The test case was disabled for `asan` in 571a647 because of the extra stack usage. Since `ubsan` also increases stack usage, seems like the two configs should be treated uniformly. On the other hand, we might be able to re-enable this test case for `asan`. During some preliminary testing on Windows, Linux, and macOS with the host compiler being as old as clang-10, the test case exited successfully if the `stack-exhausted` warnings are suppressed, though I haven't done exhaustive testing across platforms and clang versions. Any insights into whether this change will introduce any risks to existing buildbots is appreciated. Enabling this test case for `asan` helps to improve our test coverage, but if it causes problems on any buildbot, marking it as unsupported for `ubsan` is also a viable solution. --- .../SemaTemplate/instantiation-depth-default.cpp | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/clang/test/SemaTemplate/instantiation-depth-default.cpp b/clang/test/SemaTemplate/instantiation-depth-default.cpp index f5835b86b3a3..430d042d7e0f 100644 --- a/clang/test/SemaTemplate/instantiation-depth-default.cpp +++ b/clang/test/SemaTemplate/instantiation-depth-default.cpp @@ -1,18 +1,12 @@ -// RUN: %clang_cc1 -fsyntax-only -verify -ftemplate-backtrace-limit=2 %s -// -// FIXME: Disable this test when Clang was built with ASan, because ASan -// increases our per-frame stack usage enough that this test no longer fits -// within our normal stack space allocation. -// UNSUPPORTED: asan -// +// RUN: %clang_cc1 -fsyntax-only -verify -ftemplate-backtrace-limit=2 %if {{asan|ubsan}} %{ -Wno-stack-exhausted %} %s // The default stack size on NetBSD is too small for this test. // UNSUPPORTED: system-netbsd template struct X : X {}; -// expected-error-re@11 {{recursive template instantiation exceeded maximum depth of 1024{{$}}}} -// expected-note@11 {{instantiation of template class}} -// expected-note@11 {{skipping 1023 contexts in backtrace}} -// expected-note@11 {{use -ftemplate-depth=N to increase recursive template instantiation depth}} +// expected-error-re@5 {{recursive template instantiation exceeded maximum depth of 1024{{$}}}} +// expected-note@5 {{instantiation of template class}} +// expected-note@5 {{skipping 1023 contexts in backtrace}} +// expected-note@5 {{use -ftemplate-depth=N to increase recursive template instantiation depth}} X<0, int> x; // expected-note {{in instantiation of}} -- GitLab From b7e50df0262a9e563bc908e2bea06cd839a42140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Fri, 5 Jan 2024 11:54:52 +0100 Subject: [PATCH 094/728] [lli] Mimic default LLJIT logic with explicit --jit-linker option (#76964) Forcing the JITLinker to the default configuration shouldn't affect the configuration of the JIT. Since it's implemented by setting the ObjectLinkingLayerCreator, it did have side-effects though. In particular, we skipped the Code- and RelocationModel setting in `LLJITBuilderState::prepareForConstruction()`. --- llvm/tools/lli/lli.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp index 5f1fd1578764..8a7ea2d3d0c5 100644 --- a/llvm/tools/lli/lli.cpp +++ b/llvm/tools/lli/lli.cpp @@ -965,9 +965,12 @@ int runOrcJIT(const char *ProgName) { EPC = ExitOnErr(orc::SelfExecutorProcessControl::Create( std::make_shared())); - Builder.setObjectLinkingLayerCreator([&EPC, &P](orc::ExecutionSession &ES, - const Triple &TT) { - auto L = std::make_unique(ES, EPC->getMemMgr()); + Builder.getJITTargetMachineBuilder() + ->setRelocationModel(Reloc::PIC_) + .setCodeModel(CodeModel::Small); + Builder.setObjectLinkingLayerCreator([&P](orc::ExecutionSession &ES, + const Triple &TT) { + auto L = std::make_unique(ES); if (P != LLJITPlatform::ExecutorNative) L->addPlugin(std::make_unique( ES, ExitOnErr(orc::EPCEHFrameRegistrar::Create(ES)))); -- GitLab From 8c7f10eadbcf1271b6da52ccac43c241f9b8f06e Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Fri, 5 Jan 2024 11:01:09 +0000 Subject: [PATCH 095/728] [TLI] Add mappings to SLEEF/ArmPL libcall variants taking linear args. (#76060) The mappings correspond to vectorized variants (fixed/scalable) for the math functions: modf, sincos, sincospi. --- llvm/include/llvm/Analysis/VecFuncs.def | 36 +++ .../AArch64/veclib-function-calls.ll | 222 +++++++++++++++++- llvm/test/Transforms/Util/add-TLI-mappings.ll | 148 +++++++++++- 3 files changed, 403 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def index 4bffcdee6f9c..ee9207bb4f7d 100644 --- a/llvm/include/llvm/Analysis/VecFuncs.def +++ b/llvm/include/llvm/Analysis/VecFuncs.def @@ -506,12 +506,18 @@ TLI_DEFINE_VECFUNC( "llvm.log2.f64", "_ZGVnN2v_log2", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC( "log10", "_ZGVnN2v_log10", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC( "llvm.log10.f64", "_ZGVnN2v_log10", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC( "modf", "_ZGVnN2vl8_modf", FIXED(2), "_ZGV_LLVM_N2vl8") + TLI_DEFINE_VECFUNC( "pow", "_ZGVnN2vv_pow", FIXED(2), "_ZGV_LLVM_N2vv") TLI_DEFINE_VECFUNC( "llvm.pow.f64", "_ZGVnN2vv_pow", FIXED(2), "_ZGV_LLVM_N2vv") TLI_DEFINE_VECFUNC( "sin", "_ZGVnN2v_sin", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC( "llvm.sin.f64", "_ZGVnN2v_sin", FIXED(2), "_ZGV_LLVM_N2v") +TLI_DEFINE_VECFUNC( "sincos", "_ZGVnN2vl8l8_sincos", FIXED(2), "_ZGV_LLVM_N2vl8l8") + +TLI_DEFINE_VECFUNC( "sincospi", "_ZGVnN2vl8l8_sincospi", FIXED(2), "_ZGV_LLVM_N2vl8l8") + TLI_DEFINE_VECFUNC( "sinh", "_ZGVnN2v_sinh", FIXED(2), "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC( "sqrt", "_ZGVnN2v_sqrt", FIXED(2), "_ZGV_LLVM_N2v") @@ -560,12 +566,18 @@ TLI_DEFINE_VECFUNC( "llvm.log2.f32", "_ZGVnN4v_log2f", FIXED(4), "_ZGV_LLVM_N4v" TLI_DEFINE_VECFUNC( "log10f", "_ZGVnN4v_log10f", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC( "llvm.log10.f32", "_ZGVnN4v_log10f", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC( "modff", "_ZGVnN4vl4_modff", FIXED(4), "_ZGV_LLVM_N4vl4") + TLI_DEFINE_VECFUNC( "powf", "_ZGVnN4vv_powf", FIXED(4), "_ZGV_LLVM_N4vv") TLI_DEFINE_VECFUNC( "llvm.pow.f32", "_ZGVnN4vv_powf", FIXED(4), "_ZGV_LLVM_N4vv") TLI_DEFINE_VECFUNC( "sinf", "_ZGVnN4v_sinf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC( "llvm.sin.f32", "_ZGVnN4v_sinf", FIXED(4), "_ZGV_LLVM_N4v") +TLI_DEFINE_VECFUNC("sincosf", "_ZGVnN4vl4l4_sincosf", FIXED(4), "_ZGV_LLVM_N4vl4l4") + +TLI_DEFINE_VECFUNC("sincospif", "_ZGVnN4vl4l4_sincospif", FIXED(4), "_ZGV_LLVM_N4vl4l4") + TLI_DEFINE_VECFUNC( "sinhf", "_ZGVnN4v_sinhf", FIXED(4), "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC( "sqrtf", "_ZGVnN4v_sqrtf", FIXED(4), "_ZGV_LLVM_N4v") @@ -637,6 +649,9 @@ TLI_DEFINE_VECFUNC("log10f", "_ZGVsMxv_log10f", SCALABLE(4), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("llvm.log10.f64", "_ZGVsMxv_log10", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("llvm.log10.f32", "_ZGVsMxv_log10f", SCALABLE(4), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("modf", "_ZGVsMxvl8_modf", SCALABLE(2), MASKED, "_ZGVsMxvl8") +TLI_DEFINE_VECFUNC("modff", "_ZGVsMxvl4_modff", SCALABLE(4), MASKED, "_ZGVsMxvl4") + TLI_DEFINE_VECFUNC("pow", "_ZGVsMxvv_pow", SCALABLE(2), MASKED, "_ZGVsMxvv") TLI_DEFINE_VECFUNC("powf", "_ZGVsMxvv_powf", SCALABLE(4), MASKED, "_ZGVsMxvv") TLI_DEFINE_VECFUNC("llvm.pow.f64", "_ZGVsMxvv_pow", SCALABLE(2), MASKED, "_ZGVsMxvv") @@ -647,6 +662,12 @@ TLI_DEFINE_VECFUNC("sinf", "_ZGVsMxv_sinf", SCALABLE(4), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("llvm.sin.f64", "_ZGVsMxv_sin", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("llvm.sin.f32", "_ZGVsMxv_sinf", SCALABLE(4), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("sincos", "_ZGVsMxvl8l8_sincos", SCALABLE(2), MASKED, "_ZGVsMxvl8l8") +TLI_DEFINE_VECFUNC("sincosf", "_ZGVsMxvl4l4_sincosf", SCALABLE(4), MASKED, "_ZGVsMxvl4l4") + +TLI_DEFINE_VECFUNC("sincospi", "_ZGVsMxvl8l8_sincospi", SCALABLE(2), MASKED, "_ZGVsMxvl8l8") +TLI_DEFINE_VECFUNC("sincospif", "_ZGVsMxvl4l4_sincospif", SCALABLE(4), MASKED, "_ZGVsMxvl4l4") + TLI_DEFINE_VECFUNC("sinh", "_ZGVsMxv_sinh", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("sinhf", "_ZGVsMxv_sinhf", SCALABLE(4), MASKED, "_ZGVsMxv") @@ -834,6 +855,11 @@ TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_vlog10q_f32", FIXED(4), NOMASK, "_ZG TLI_DEFINE_VECFUNC("llvm.log10.f64", "armpl_svlog10_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("llvm.log10.f32", "armpl_svlog10_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("modf", "armpl_vmodfq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8") +TLI_DEFINE_VECFUNC("modff", "armpl_vmodfq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4") +TLI_DEFINE_VECFUNC("modf", "armpl_svmodf_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8") +TLI_DEFINE_VECFUNC("modff", "armpl_svmodf_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4") + TLI_DEFINE_VECFUNC("nextafter", "armpl_vnextafterq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vv") TLI_DEFINE_VECFUNC("nextafterf", "armpl_vnextafterq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vv") TLI_DEFINE_VECFUNC("nextafter", "armpl_svnextafter_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvv") @@ -859,6 +885,16 @@ TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_vsinq_f32", FIXED(4), NOMASK, "_ZGV_LL TLI_DEFINE_VECFUNC("llvm.sin.f64", "armpl_svsin_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv") TLI_DEFINE_VECFUNC("llvm.sin.f32", "armpl_svsin_f32_x", SCALABLE(4), MASKED, "_ZGVsMxv") +TLI_DEFINE_VECFUNC("sincos", "armpl_vsincosq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8") +TLI_DEFINE_VECFUNC("sincosf", "armpl_vsincosq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4") +TLI_DEFINE_VECFUNC("sincos", "armpl_svsincos_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8l8") +TLI_DEFINE_VECFUNC("sincosf", "armpl_svsincos_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4") + +TLI_DEFINE_VECFUNC("sincospi", "armpl_vsincospiq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2vl8l8") +TLI_DEFINE_VECFUNC("sincospif", "armpl_vsincospiq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4vl4l4") +TLI_DEFINE_VECFUNC("sincospi", "armpl_svsincospi_f64_x", SCALABLE(2), MASKED, "_ZGVsMxvl8l8") +TLI_DEFINE_VECFUNC("sincospif", "armpl_svsincospi_f32_x", SCALABLE(4), MASKED, "_ZGVsMxvl4l4") + TLI_DEFINE_VECFUNC("sinh", "armpl_vsinhq_f64", FIXED(2), NOMASK, "_ZGV_LLVM_N2v") TLI_DEFINE_VECFUNC("sinhf", "armpl_vsinhq_f32", FIXED(4), NOMASK, "_ZGV_LLVM_N4v") TLI_DEFINE_VECFUNC("sinh", "armpl_svsinh_f64_x", SCALABLE(2), MASKED, "_ZGVsMxv") diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll index ebee4fa42e9b..e30173a58881 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-function-calls.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|sin|tan|cbrt|erf|exp|gamma|log|sqrt|copysign|dim|min|mod|hypot|nextafter|pow|fma)" --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(cos|sin|tan|cbrt|erf|exp[^e]|gamma|log|sqrt|copysign|dim|min|mod|hypot|nextafter|pow|fma)" --version 2 ; RUN: opt -mattr=+neon -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SLEEF-NEON ; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s --check-prefix=SLEEF-SVE ; RUN: opt -mattr=+neon -vector-library=ArmPL -passes=inject-tli-mappings,loop-vectorize,simplifycfg -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=ARMPL-NEON @@ -2639,3 +2639,223 @@ define void @fma_f32(ptr noalias %in.ptr, ptr noalias %out.ptr) { ret void } +declare double @modf(double, ptr) +declare float @modff(float, ptr) + +define void @test_modf(ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; SLEEF-NEON-LABEL: define void @test_modf +; SLEEF-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-NEON: [[TMP5:%.*]] = call <2 x double> @_ZGVnN2vl8_modf(<2 x double> [[WIDE_LOAD:%.*]], ptr [[TMP4:%.*]]) +; +; SLEEF-SVE-LABEL: define void @test_modf +; SLEEF-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-SVE: [[TMP23:%.*]] = call @_ZGVsMxvl8_modf( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP22:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @test_modf +; ARMPL-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-NEON: [[TMP5:%.*]] = call <2 x double> @armpl_vmodfq_f64(<2 x double> [[WIDE_LOAD:%.*]], ptr [[TMP4:%.*]]) +; +; ARMPL-SVE-LABEL: define void @test_modf +; ARMPL-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-SVE: [[TMP23:%.*]] = call @armpl_svmodf_f64_x( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP22:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %gepa = getelementptr double, ptr %a, i64 %indvars.iv + %num = load double, ptr %gepa, align 8 + %gepb = getelementptr double, ptr %b, i64 %indvars.iv + %data = call double @modf(double %num, ptr %gepb) + %gepc = getelementptr inbounds double, ptr %c, i64 %indvars.iv + store double %data, ptr %gepc, align 8 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1000 + br i1 %exitcond, label %for.cond.cleanup, label %for.body + +for.cond.cleanup: + ret void +} + +define void @test_modff(ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; SLEEF-NEON-LABEL: define void @test_modff +; SLEEF-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-NEON: [[TMP5:%.*]] = call <4 x float> @_ZGVnN4vl4_modff(<4 x float> [[WIDE_LOAD:%.*]], ptr [[TMP4:%.*]]) +; +; SLEEF-SVE-LABEL: define void @test_modff +; SLEEF-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-SVE: [[TMP23:%.*]] = call @_ZGVsMxvl4_modff( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP22:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @test_modff +; ARMPL-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-NEON: [[TMP5:%.*]] = call <4 x float> @armpl_vmodfq_f32(<4 x float> [[WIDE_LOAD:%.*]], ptr [[TMP4:%.*]]) +; +; ARMPL-SVE-LABEL: define void @test_modff +; ARMPL-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-SVE: [[TMP23:%.*]] = call @armpl_svmodf_f32_x( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP22:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %gepa = getelementptr float, ptr %a, i64 %indvars.iv + %num = load float, ptr %gepa, align 8 + %gepb = getelementptr float, ptr %b, i64 %indvars.iv + %data = call float @modff(float %num, ptr %gepb) + %gepc = getelementptr inbounds float, ptr %c, i64 %indvars.iv + store float %data, ptr %gepc, align 8 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1000 + br i1 %exitcond, label %for.cond.cleanup, label %for.body + +for.cond.cleanup: + ret void +} + +declare void @sincos(double, ptr, ptr) +declare void @sincosf(float, ptr, ptr) + +define void @test_sincos(ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; SLEEF-NEON-LABEL: define void @test_sincos +; SLEEF-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-NEON: call void @_ZGVnN2vl8l8_sincos(<2 x double> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; SLEEF-SVE-LABEL: define void @test_sincos +; SLEEF-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-SVE: call void @_ZGVsMxvl8l8_sincos( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @test_sincos +; ARMPL-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-NEON: call void @armpl_vsincosq_f64(<2 x double> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; ARMPL-SVE-LABEL: define void @test_sincos +; ARMPL-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-SVE: call void @armpl_svsincos_f64_x( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %gepa = getelementptr double, ptr %a, i64 %indvars.iv + %num = load double, ptr %gepa, align 8 + %gepb = getelementptr double, ptr %b, i64 %indvars.iv + %gepc = getelementptr double, ptr %c, i64 %indvars.iv + call void @sincos(double %num, ptr %gepb, ptr %gepc) + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1000 + br i1 %exitcond, label %for.cond.cleanup, label %for.body + +for.cond.cleanup: + ret void +} + +define void @test_sincosf(ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; SLEEF-NEON-LABEL: define void @test_sincosf +; SLEEF-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-NEON: call void @_ZGVnN4vl4l4_sincosf(<4 x float> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; SLEEF-SVE-LABEL: define void @test_sincosf +; SLEEF-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-SVE: call void @_ZGVsMxvl4l4_sincosf( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @test_sincosf +; ARMPL-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-NEON: call void @armpl_vsincosq_f32(<4 x float> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; ARMPL-SVE-LABEL: define void @test_sincosf +; ARMPL-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-SVE: call void @armpl_svsincos_f32_x( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %gepa = getelementptr float, ptr %a, i64 %indvars.iv + %num = load float, ptr %gepa, align 8 + %gepb = getelementptr float, ptr %b, i64 %indvars.iv + %gepc = getelementptr float, ptr %c, i64 %indvars.iv + call void @sincosf(float %num, ptr %gepb, ptr %gepc) + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1000 + br i1 %exitcond, label %for.cond.cleanup, label %for.body + +for.cond.cleanup: + ret void +} + +declare void @sincospi(double, ptr, ptr) +declare void @sincospif(float, ptr, ptr) + +define void @test_sincospi(ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; SLEEF-NEON-LABEL: define void @test_sincospi +; SLEEF-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-NEON: call void @_ZGVnN2vl8l8_sincospi(<2 x double> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; SLEEF-SVE-LABEL: define void @test_sincospi +; SLEEF-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-SVE: call void @_ZGVsMxvl8l8_sincospi( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @test_sincospi +; ARMPL-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-NEON: call void @armpl_vsincospiq_f64(<2 x double> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; ARMPL-SVE-LABEL: define void @test_sincospi +; ARMPL-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-SVE: call void @armpl_svsincospi_f64_x( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %gepa = getelementptr double, ptr %a, i64 %indvars.iv + %num = load double, ptr %gepa, align 8 + %gepb = getelementptr double, ptr %b, i64 %indvars.iv + %gepc = getelementptr double, ptr %c, i64 %indvars.iv + call void @sincospi(double %num, ptr %gepb, ptr %gepc) + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1000 + br i1 %exitcond, label %for.cond.cleanup, label %for.body + +for.cond.cleanup: + ret void +} + +define void @test_sincospif(ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; SLEEF-NEON-LABEL: define void @test_sincospif +; SLEEF-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-NEON: call void @_ZGVnN4vl4l4_sincospif(<4 x float> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; SLEEF-SVE-LABEL: define void @test_sincospif +; SLEEF-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; SLEEF-SVE: call void @_ZGVsMxvl4l4_sincospif( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +; ARMPL-NEON-LABEL: define void @test_sincospif +; ARMPL-NEON-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-NEON: call void @armpl_vsincospiq_f32(<4 x float> [[WIDE_LOAD:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]]) +; +; ARMPL-SVE-LABEL: define void @test_sincospif +; ARMPL-SVE-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; ARMPL-SVE: call void @armpl_svsincospi_f32_x( [[WIDE_MASKED_LOAD:%.*]], ptr [[TMP23:%.*]], ptr [[TMP24:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %gepa = getelementptr float, ptr %a, i64 %indvars.iv + %num = load float, ptr %gepa, align 8 + %gepb = getelementptr float, ptr %b, i64 %indvars.iv + %gepc = getelementptr float, ptr %c, i64 %indvars.iv + call void @sincospif(float %num, ptr %gepb, ptr %gepc) + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1000 + br i1 %exitcond, label %for.cond.cleanup, label %for.body + +for.cond.cleanup: + ret void +} diff --git a/llvm/test/Transforms/Util/add-TLI-mappings.ll b/llvm/test/Transforms/Util/add-TLI-mappings.ll index 67ca00b8e2cf..8ccd27030afa 100644 --- a/llvm/test/Transforms/Util/add-TLI-mappings.ll +++ b/llvm/test/Transforms/Util/add-TLI-mappings.ll @@ -21,18 +21,62 @@ ; LIBMVEC-X86-SAME: [2 x ptr] [ ; LIBMVEC-X86-SAME: ptr @_ZGVbN2v_sin, ; LIBMVEC-X86-SAME: ptr @_ZGVdN4v_sin -; SLEEFGNUABI-SAME: [4 x ptr] [ +; SLEEFGNUABI-SAME: [16 x ptr] [ +; SLEEFGNUABI-SAME: ptr @_ZGVnN2vl8_modf, +; SLEEFGNUABI-SAME: ptr @_ZGVsMxvl8_modf, +; SLEEFGNUABI-SAME: ptr @_ZGVnN4vl4_modff, +; SLEEFGNUABI-SAME: ptr @_ZGVsMxvl4_modff, ; SLEEFGNUABI-SAME: ptr @_ZGVnN2v_sin, ; SLEEFGNUABI-SAME: ptr @_ZGVsMxv_sin, +; SLEEFGNUABI-SAME: ptr @_ZGVnN2vl8l8_sincos, +; SLEEFGNUABI-SAME: ptr @_ZGVsMxvl8l8_sincos, +; SLEEFGNUABI-SAME: ptr @_ZGVnN4vl4l4_sincosf, +; SLEEFGNUABI-SAME: ptr @_ZGVsMxvl4l4_sincosf, +; SLEEFGNUABI-SAME: ptr @_ZGVnN2vl8l8_sincospi, +; SLEEFGNUABI-SAME: ptr @_ZGVsMxvl8l8_sincospi, +; SLEEFGNUABI-SAME: ptr @_ZGVnN4vl4l4_sincospif, +; SLEEFGNUABI-SAME: ptr @_ZGVsMxvl4l4_sincospif, ; SLEEFGNUABI_SAME; ptr @_ZGVnN4v_log10f, ; SLEEFGNUABI-SAME: ptr @_ZGVsMxv_log10f -; ARMPL-SAME: [4 x ptr] [ +; ARMPL-SAME: [16 x ptr] [ +; ARMPL-SAME: ptr @armpl_vmodfq_f64, +; ARMPL-SAME: ptr @armpl_svmodf_f64_x, +; ARMPL-SAME: ptr @armpl_vmodfq_f32, +; ARMPL-SAME: ptr @armpl_svmodf_f32_x, ; ARMPL-SAME: ptr @armpl_vsinq_f64, ; ARMPL-SAME: ptr @armpl_svsin_f64_x, +; ARMPL-SAME: ptr @armpl_vsincosq_f64, +; ARMPL-SAME: ptr @armpl_svsincos_f64_x, +; ARMPL-SAME: ptr @armpl_vsincosq_f32, +; ARMPL-SAME: ptr @armpl_svsincos_f32_x, +; ARMPL-SAME: ptr @armpl_vsincospiq_f64, +; ARMPL-SAME: ptr @armpl_svsincospi_f64_x, +; ARMPL-SAME: ptr @armpl_vsincospiq_f32, +; ARMPL-SAME: ptr @armpl_svsincospi_f32_x, ; ARMPL-SAME: ptr @armpl_vlog10q_f32, ; ARMPL-SAME: ptr @armpl_svlog10_f32_x ; COMMON-SAME: ], section "llvm.metadata" +define double @modf_f64(double %in, ptr %iptr) { +; COMMON-LABEL: @modf_f64( +; SLEEFGNUABI: call double @modf(double %{{.*}}, ptr %{{.*}}) #[[MODF:[0-9]+]] +; ARMPL: call double @modf(double %{{.*}}, ptr %{{.*}}) #[[MODF:[0-9]+]] + %call = tail call double @modf(double %in, ptr %iptr) + ret double %call +} + +declare double @modf(double, ptr) #0 + +define float @modf_f32(float %in, ptr %iptr) { +; COMMON-LABEL: @modf_f32( +; SLEEFGNUABI: call float @modff(float %{{.*}}, ptr %{{.*}}) #[[MODFF:[0-9]+]] +; ARMPL: call float @modff(float %{{.*}}, ptr %{{.*}}) #[[MODFF:[0-9]+]] + %call = tail call float @modff(float %in, ptr %iptr) + ret float %call +} + +declare float @modff(float, ptr) #0 + define double @sin_f64(double %in) { ; COMMON-LABEL: @sin_f64( ; SVML: call double @sin(double %{{.*}}) #[[SIN:[0-9]+]] @@ -49,6 +93,46 @@ define double @sin_f64(double %in) { declare double @sin(double) #0 +define void @sincos_f64(double %in, ptr %sin, ptr %cos) { +; COMMON-LABEL: @sincos_f64( +; SLEEFGNUABI: call void @sincos(double %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOS:[0-9]+]] +; ARMPL: call void @sincos(double %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOS:[0-9]+]] + call void @sincos(double %in, ptr %sin, ptr %cos) + ret void +} + +declare void @sincos(double, ptr, ptr) #0 + +define void @sincos_f32(float %in, ptr %sin, ptr %cos) { +; COMMON-LABEL: @sincos_f32( +; SLEEFGNUABI: call void @sincosf(float %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSF:[0-9]+]] +; ARMPL: call void @sincosf(float %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSF:[0-9]+]] + call void @sincosf(float %in, ptr %sin, ptr %cos) + ret void +} + +declare void @sincosf(float, ptr, ptr) #0 + +define void @sincospi_f64(double %in, ptr %sin, ptr %cos) { +; COMMON-LABEL: @sincospi_f64( +; SLEEFGNUABI: call void @sincospi(double %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSPI:[0-9]+]] +; ARMPL: call void @sincospi(double %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSPI:[0-9]+]] + call void @sincospi(double %in, ptr %sin, ptr %cos) + ret void +} + +declare void @sincospi(double, ptr, ptr) #0 + +define void @sincospi_f32(float %in, ptr %sin, ptr %cos) { +; COMMON-LABEL: @sincospi_f32( +; SLEEFGNUABI: call void @sincospif(float %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSPIF:[0-9]+]] +; ARMPL: call void @sincospif(float %{{.*}}, ptr %{{.*}}, ptr %{{.*}}) #[[SINCOSPIF:[0-9]+]] + call void @sincospif(float %in, ptr %sin, ptr %cos) + ret void +} + +declare void @sincospif(float, ptr, ptr) #0 + define float @call_llvm.log10.f32(float %in) { ; COMMON-LABEL: @call_llvm.log10.f32( ; SVML: call float @llvm.log10.f32(float %{{.*}}) @@ -81,13 +165,37 @@ declare float @llvm.log10.f32(float) #0 ; ACCELERATE: declare <4 x float> @vlog10f(<4 x float>) +; SLEEFGNUABI: declare <2 x double> @_ZGVnN2vl8_modf(<2 x double>, ptr) +; SLEEFGNUABI: declare @_ZGVsMxvl8_modf(, ptr, ) +; SLEEFGNUABI: declare <4 x float> @_ZGVnN4vl4_modff(<4 x float>, ptr) +; SLEEFGNUABI: declare @_ZGVsMxvl4_modff(, ptr, ) ; SLEEFGNUABI: declare <2 x double> @_ZGVnN2v_sin(<2 x double>) ; SLEEFGNUABI: declare @_ZGVsMxv_sin(, ) +; SLEEFGNUABI: declare void @_ZGVnN2vl8l8_sincos(<2 x double>, ptr, ptr) +; SLEEFGNUABI: declare void @_ZGVsMxvl8l8_sincos(, ptr, ptr, ) +; SLEEFGNUABI: declare void @_ZGVnN4vl4l4_sincosf(<4 x float>, ptr, ptr) +; SLEEFGNUABI: declare void @_ZGVsMxvl4l4_sincosf(, ptr, ptr, ) +; SLEEFGNUABI: declare void @_ZGVnN2vl8l8_sincospi(<2 x double>, ptr, ptr) +; SLEEFGNUABI: declare void @_ZGVsMxvl8l8_sincospi(, ptr, ptr, ) +; SLEEFGNUABI: declare void @_ZGVnN4vl4l4_sincospif(<4 x float>, ptr, ptr) +; SLEEFGNUABI: declare void @_ZGVsMxvl4l4_sincospif(, ptr, ptr, ) ; SLEEFGNUABI: declare <4 x float> @_ZGVnN4v_log10f(<4 x float>) ; SLEEFGNUABI: declare @_ZGVsMxv_log10f(, ) +; ARMPL: declare <2 x double> @armpl_vmodfq_f64(<2 x double>, ptr) +; ARMPL: declare @armpl_svmodf_f64_x(, ptr, ) +; ARMPL: declare <4 x float> @armpl_vmodfq_f32(<4 x float>, ptr) +; ARMPL: declare @armpl_svmodf_f32_x(, ptr, ) ; ARMPL: declare <2 x double> @armpl_vsinq_f64(<2 x double>) ; ARMPL: declare @armpl_svsin_f64_x(, ) +; ARMPL: declare void @armpl_vsincosq_f64(<2 x double>, ptr, ptr) +; ARMPL: declare void @armpl_svsincos_f64_x(, ptr, ptr, ) +; ARMPL: declare void @armpl_vsincosq_f32(<4 x float>, ptr, ptr) +; ARMPL: declare void @armpl_svsincos_f32_x(, ptr, ptr, ) +; ARMPL: declare void @armpl_vsincospiq_f64(<2 x double>, ptr, ptr) +; ARMPL: declare void @armpl_svsincospi_f64_x(, ptr, ptr, ) +; ARMPL: declare void @armpl_vsincospiq_f32(<4 x float>, ptr, ptr) +; ARMPL: declare void @armpl_svsincospi_f32_x(, ptr, ptr, ) ; ARMPL: declare <4 x float> @armpl_vlog10q_f32(<4 x float>) ; ARMPL: declare @armpl_svlog10_f32_x(, ) @@ -110,16 +218,52 @@ attributes #0 = { nounwind readnone } ; LIBMVEC-X86-SAME: "_ZGV_LLVM_N2v_sin(_ZGVbN2v_sin), ; LIBMVEC-X86-SAME: _ZGV_LLVM_N4v_sin(_ZGVdN4v_sin)" } +; SLEEFGNUABI: attributes #[[MODF]] = { "vector-function-abi-variant"= +; SLEEFGNUABI-SAME: "_ZGV_LLVM_N2vl8_modf(_ZGVnN2vl8_modf), +; SLEEFGNUABI-SAME: _ZGVsMxvl8_modf(_ZGVsMxvl8_modf)" } +; SLEEFGNUABI: attributes #[[MODFF]] = { "vector-function-abi-variant"= +; SLEEFGNUABI-SAME: "_ZGV_LLVM_N4vl4_modff(_ZGVnN4vl4_modff), +; SLEEFGNUABI-SAME: _ZGVsMxvl4_modff(_ZGVsMxvl4_modff)" } ; SLEEFGNUABI: attributes #[[SIN]] = { "vector-function-abi-variant"= ; SLEEFGNUABI-SAME: "_ZGV_LLVM_N2v_sin(_ZGVnN2v_sin), ; SLEEFGNUABI-SAME: _ZGVsMxv_sin(_ZGVsMxv_sin)" } +; SLEEFGNUABI: attributes #[[SINCOS]] = { "vector-function-abi-variant"= +; SLEEFGNUABI-SAME: "_ZGV_LLVM_N2vl8l8_sincos(_ZGVnN2vl8l8_sincos), +; SLEEFGNUABI-SAME: _ZGVsMxvl8l8_sincos(_ZGVsMxvl8l8_sincos)" } +; SLEEFGNUABI: attributes #[[SINCOSF]] = { "vector-function-abi-variant"= +; SLEEFGNUABI-SAME: "_ZGV_LLVM_N4vl4l4_sincosf(_ZGVnN4vl4l4_sincosf), +; SLEEFGNUABI-SAME: _ZGVsMxvl4l4_sincosf(_ZGVsMxvl4l4_sincosf)" } +; SLEEFGNUABI: attributes #[[SINCOSPI]] = { "vector-function-abi-variant"= +; SLEEFGNUABI-SAME: "_ZGV_LLVM_N2vl8l8_sincospi(_ZGVnN2vl8l8_sincospi), +; SLEEFGNUABI-SAME: _ZGVsMxvl8l8_sincospi(_ZGVsMxvl8l8_sincospi)" } +; SLEEFGNUABI: attributes #[[SINCOSPIF]] = { "vector-function-abi-variant"= +; SLEEFGNUABI-SAME: "_ZGV_LLVM_N4vl4l4_sincospif(_ZGVnN4vl4l4_sincospif), +; SLEEFGNUABI-SAME: _ZGVsMxvl4l4_sincospif(_ZGVsMxvl4l4_sincospif)" } ; SLEEFGNUABI: attributes #[[LOG10]] = { "vector-function-abi-variant"= ; SLEEFGNUABI-SAME: "_ZGV_LLVM_N4v_llvm.log10.f32(_ZGVnN4v_log10f), ; SLEEFGNUABI-SAME: _ZGVsMxv_llvm.log10.f32(_ZGVsMxv_log10f)" } +; ARMPL: attributes #[[MODF]] = { "vector-function-abi-variant"= +; ARMPL-SAME: "_ZGV_LLVM_N2vl8_modf(armpl_vmodfq_f64), +; ARMPL-SAME: _ZGVsMxvl8_modf(armpl_svmodf_f64_x)" } +; ARMPL: attributes #[[MODFF]] = { "vector-function-abi-variant"= +; ARMPL-SAME: "_ZGV_LLVM_N4vl4_modff(armpl_vmodfq_f32), +; ARMPL-SAME: _ZGVsMxvl4_modff(armpl_svmodf_f32_x)" } ; ARMPL: attributes #[[SIN]] = { "vector-function-abi-variant"= ; ARMPL-SAME: "_ZGV_LLVM_N2v_sin(armpl_vsinq_f64), ; ARMPL-SAME _ZGVsMxv_sin(armpl_svsin_f64_x)" } +; ARMPL: attributes #[[SINCOS]] = { "vector-function-abi-variant"= +; ARMPL-SAME: "_ZGV_LLVM_N2vl8l8_sincos(armpl_vsincosq_f64), +; ARMPL-SAME: _ZGVsMxvl8l8_sincos(armpl_svsincos_f64_x)" } +; ARMPL: attributes #[[SINCOSF]] = { "vector-function-abi-variant"= +; ARMPL-SAME: "_ZGV_LLVM_N4vl4l4_sincosf(armpl_vsincosq_f32), +; ARMPL-SAME: _ZGVsMxvl4l4_sincosf(armpl_svsincos_f32_x)" } +; ARMPL: attributes #[[SINCOSPI]] = { "vector-function-abi-variant"= +; ARMPL-SAME: "_ZGV_LLVM_N2vl8l8_sincospi(armpl_vsincospiq_f64), +; ARMPL-SAME: _ZGVsMxvl8l8_sincospi(armpl_svsincospi_f64_x)" } +; ARMPL: attributes #[[SINCOSPIF]] = { "vector-function-abi-variant"= +; ARMPL-SAME: "_ZGV_LLVM_N4vl4l4_sincospif(armpl_vsincospiq_f32), +; ARMPL-SAME: _ZGVsMxvl4l4_sincospif(armpl_svsincospi_f32_x)" } ; ARMPL: attributes #[[LOG10]] = { "vector-function-abi-variant"= ; ARMPL-SAME: "_ZGV_LLVM_N4v_llvm.log10.f32(armpl_vlog10q_f32), ; ARMPL-SAME _ZGVsMxv_llvm.log10.f32(armpl_svlog10_f32_x)" } -- GitLab From 567941bcc3b1fc3b1d2a902cf7ae2e173247a45f Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Fri, 5 Jan 2024 11:08:28 +0000 Subject: [PATCH 096/728] [Clang][SME] Remove unused HasSVE2p1 variable Removes the HasSVE2p1 variable to stop a warning from https://github.com/llvm/llvm-project/pull/76975 --- clang/lib/Basic/Targets/AArch64.h | 1 - 1 file changed, 1 deletion(-) diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h index e38fa5af5659..f0e0782e7abe 100644 --- a/clang/lib/Basic/Targets/AArch64.h +++ b/clang/lib/Basic/Targets/AArch64.h @@ -50,7 +50,6 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo { bool HasMatMul = false; bool HasBFloat16 = false; bool HasSVE2 = false; - bool HasSVE2p1 = false; bool HasSVE2AES = false; bool HasSVE2SHA3 = false; bool HasSVE2SM4 = false; -- GitLab From 44b3cf46e992b76a9e188367922a1184437dfa4c Mon Sep 17 00:00:00 2001 From: drazi Date: Fri, 5 Jan 2024 19:37:24 +0800 Subject: [PATCH 097/728] add prop-dict support for custom directive for mlir-tblgen (#77061) According to https://mlir.llvm.org/docs/DefiningDialects/Operations/#custom-directives, custom directive supports attr-dict > attr-dict Directive: NamedAttrList & But it doesn't support prop-dict which is introduced into MLIR recently. It's useful to have tblgen support prop-dict like attr-dict. This PR enable tblgen to support prop-dict ```bash error: only variables and types may be used as parameters to a custom directive ... custom(prop-dict) ``` Co-authored-by: Fung Xie --- mlir/test/mlir-tblgen/op-format-spec.td | 3 +++ mlir/test/mlir-tblgen/op-format.td | 8 ++++++++ mlir/tools/mlir-tblgen/OpFormatGen.cpp | 11 ++++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/mlir/test/mlir-tblgen/op-format-spec.td b/mlir/test/mlir-tblgen/op-format-spec.td index 5d786cce5f28..ad2a055bc78b 100644 --- a/mlir/test/mlir-tblgen/op-format-spec.td +++ b/mlir/test/mlir-tblgen/op-format-spec.td @@ -44,6 +44,9 @@ def DirectiveCustomValidC : TestFormat_Op<[{ def DirectiveCustomValidD : TestFormat_Op<[{ (`(` custom($operand)^ `)`)? attr-dict }]>, Arguments<(ins Optional:$operand)>; +def DirectiveCustomValidE : TestFormat_Op<[{ + custom(prop-dict) attr-dict +}]>, Arguments<(ins UnitAttr:$flag)>; //===----------------------------------------------------------------------===// // functional-type diff --git a/mlir/test/mlir-tblgen/op-format.td b/mlir/test/mlir-tblgen/op-format.td index dd756788098c..3250589605f3 100644 --- a/mlir/test/mlir-tblgen/op-format.td +++ b/mlir/test/mlir-tblgen/op-format.td @@ -42,6 +42,14 @@ def CustomStringLiteralC : TestFormat_Op<[{ custom("$_builder.getStringAttr(\"foo\")") attr-dict }]>; +// CHECK-LABEL: CustomStringLiteralD::parse +// CHECK: parseFoo({{.*}}, result) +// CHECK-LABEL: CustomStringLiteralD::print +// CHECK: printFoo({{.*}}, getProperties()) +def CustomStringLiteralD : TestFormat_Op<[{ + custom(prop-dict) attr-dict +}]>; + //===----------------------------------------------------------------------===// // Optional Groups //===----------------------------------------------------------------------===// diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp index 8553920f7713..b6405baef28e 100644 --- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp +++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp @@ -895,6 +895,8 @@ static void genCustomParameterParser(FormatElement *param, MethodBody &body) { body << attr->getVar()->name << "Attr"; } else if (isa(param)) { body << "result.attributes"; + } else if (isa(param)) { + body << "result"; } else if (auto *operand = dyn_cast(param)) { StringRef name = operand->getVar()->name; ArgumentLengthKind lengthKind = getArgumentLengthKind(operand->getVar()); @@ -1854,6 +1856,9 @@ static void genCustomDirectiveParameterPrinter(FormatElement *element, } else if (isa(element)) { body << "getOperation()->getAttrDictionary()"; + } else if (isa(element)) { + body << "getProperties()"; + } else if (auto *operand = dyn_cast(element)) { body << op.getGetterName(operand->getVar()->name) << "()"; @@ -3138,9 +3143,9 @@ OpFormatParser::parsePropDictDirective(SMLoc loc, Context context) { LogicalResult OpFormatParser::verifyCustomDirectiveArguments( SMLoc loc, ArrayRef arguments) { for (FormatElement *argument : arguments) { - if (!isa(argument)) { + if (!isa(argument)) { // TODO: FormatElement should have location info attached. return emitError(loc, "only variables and types may be used as " "parameters to a custom directive"); -- GitLab From ace69e6b942b8fa7e610d70be2a92e801ceea481 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Fri, 5 Jan 2024 12:51:25 +0100 Subject: [PATCH 098/728] [mlir][gpu] Improve `gpu-lower-to-nvvm-pipeline` Documentation (#77062) This PR improves the documentation for the `gpu-lower-to-nvvm-pipeline` (as it was remaning item for #75775) - Changes pipeline `gpu-lower-to-nvvm` -> `gpu-lower-to-nvvm-pipeline` - Adds a section in GPU Dialect in website. It clarifies the pipeline's functionality in lowering primary dialects to NVVM targets. --- mlir/docs/Dialects/GPU.md | 44 ++++ .../mlir/Dialect/GPU/Pipelines/Passes.h | 56 +++++ .../GPU/Pipelines/GPUToNVVMPipeline.cpp | 78 ++----- .../SparseTensor/GPU/CUDA/dump-ptx.mlir | 2 +- .../GPU/CUDA/sparse-mma-2-4-f16.mlir | 2 +- .../GPU/CUDA/test-reduction-distribute.mlir | 2 +- .../Vector/GPU/CUDA/test-warp-distribute.mlir | 6 +- ...ansform-mma-sync-matmul-f16-f16-accum.mlir | 2 +- .../sm80/transform-mma-sync-matmul-f32.mlir | 2 +- .../GPU/CUDA/TensorCore/wmma-matmul-f16.mlir | 2 +- .../TensorCore/wmma-matmul-f32-bare-ptr.mlir | 2 +- .../GPU/CUDA/TensorCore/wmma-matmul-f32.mlir | 2 +- .../Integration/GPU/CUDA/all-reduce-and.mlir | 4 +- .../GPU/CUDA/all-reduce-maxsi.mlir | 2 +- .../GPU/CUDA/all-reduce-minsi.mlir | 2 +- .../Integration/GPU/CUDA/all-reduce-op.mlir | 2 +- .../Integration/GPU/CUDA/all-reduce-or.mlir | 2 +- .../GPU/CUDA/all-reduce-region.mlir | 2 +- .../Integration/GPU/CUDA/all-reduce-xor.mlir | 2 +- .../Integration/GPU/CUDA/gpu-to-cubin.mlir | 2 +- .../GPU/CUDA/multiple-all-reduce.mlir | 2 +- mlir/test/Integration/GPU/CUDA/printf.mlir | 2 +- mlir/test/Integration/GPU/CUDA/shuffle.mlir | 2 +- mlir/test/Integration/GPU/CUDA/sm90/asd | 207 ++++++++++++++++++ .../GPU/CUDA/sm90/cga_cluster.mlir | 2 +- .../sm90/gemm_f32_f16_f16_128x128x128.mlir | 3 +- .../gemm_pred_f32_f16_f16_128x128x128.mlir | 3 +- .../sm90/tma_load_128x64_swizzle128b.mlir | 2 +- .../CUDA/sm90/tma_load_64x64_swizzle128b.mlir | 2 +- .../sm90/tma_load_64x8_8x128_noswizzle.mlir | 2 +- .../Integration/GPU/CUDA/two-modules.mlir | 2 +- 31 files changed, 354 insertions(+), 93 deletions(-) create mode 100644 mlir/test/Integration/GPU/CUDA/sm90/asd diff --git a/mlir/docs/Dialects/GPU.md b/mlir/docs/Dialects/GPU.md index 8558667ea51a..85255fdc5e64 100644 --- a/mlir/docs/Dialects/GPU.md +++ b/mlir/docs/Dialects/GPU.md @@ -60,6 +60,50 @@ mlir-translate example-nvvm.mlir \ -o example.ll ``` +### Default NVVM Compilation Pipeline: gpu-lower-to-nvvm-pipeline + +The `gpu-lower-to-nvvm-pipeline` compilation pipeline serves as the default way +for NVVM target compilation within MLIR. This pipeline operates by lowering +primary dialects (arith, memref, scf, vector, gpu, and nvgpu) to NVVM target. It +begins by lowering GPU code region(s) to the specified NVVM compilation target +and subsequently handles the host code. + +This pipeline specifically requires explicitly parallel IR and doesn't do GPU +parallelization. To enable parallelism, necessary transformations must be +applied before utilizing this pipeline. + +It's designed to provide a generic solution for NVVM targets, generating NVVM +and LLVM dialect code compatible with `mlir-cpu-runner` or execution engine. + +#### Example: + +Here's a snippet illustrating the use of primary dialects, including arith, +within GPU code execution: + +``` +func.func @main() { + %c2 = arith.constant 2 : index + %c1 = arith.constant 1 : index + gpu.launch + blocks(%0, %1, %2) in (%3 = %c1, %4 = %c1, %5 = %c1) + threads(%6, %7, %8) in (%9 = %c2, %10 = %c1, %11 = %c1) { + gpu.printf "Hello from %d\n" %6 : index + gpu.terminator + } + return +} +``` + +The `gpu-lower-to-nvvm` pipeline compiles this input code to NVVM format as +below. It provides customization options like specifying SM capability, PTX +version, and optimization level. Once compiled, the resulting IR is ready for +execution using `mlir-cpu-runner`. Alternatively, it can be translated into +LLVM, expanding its utility within the system. + +``` +mlir-opt example.mlir -gpu-lower-to-nvvm-pipeline = "cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3" +``` + ### Module serialization Attributes implementing the GPU Target Attribute Interface handle the serialization process and are called Target attributes. These attributes can be diff --git a/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h b/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h index 7128ffff2b74..caa0901bb494 100644 --- a/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h +++ b/mlir/include/mlir/Dialect/GPU/Pipelines/Passes.h @@ -9,9 +9,65 @@ #ifndef MLIR_DIALECT_GPU_PIPELINES_PASSES_H_ #define MLIR_DIALECT_GPU_PIPELINES_PASSES_H_ +#include "mlir/Pass/PassOptions.h" + namespace mlir { namespace gpu { + +/// Options for the gpu to nvvm pipeline. +struct GPUToNVVMPipelineOptions + : public PassPipelineOptions { + PassOptions::Option indexBitWidth{ + *this, "index-bitwidth", + llvm::cl::desc("Bitwidth of the index type for the host (warning this " + "should be 64 until the GPU layering is fixed)"), + llvm::cl::init(64)}; + PassOptions::Option cubinTriple{ + *this, "cubin-triple", + llvm::cl::desc("Triple to use to serialize to cubin."), + llvm::cl::init("nvptx64-nvidia-cuda")}; + PassOptions::Option cubinChip{ + *this, "cubin-chip", llvm::cl::desc("Chip to use to serialize to cubin."), + llvm::cl::init("sm_50")}; + PassOptions::Option cubinFeatures{ + *this, "cubin-features", + llvm::cl::desc("Features to use to serialize to cubin."), + llvm::cl::init("+ptx60")}; + PassOptions::Option cubinFormat{ + *this, "cubin-format", + llvm::cl::desc("Compilation format to use to serialize to cubin."), + llvm::cl::init("fatbin")}; + PassOptions::Option optLevel{ + *this, "opt-level", + llvm::cl::desc("Optimization level for NVVM compilation"), + llvm::cl::init(2)}; + PassOptions::Option kernelUseBarePtrCallConv{ + *this, "kernel-bare-ptr-calling-convention", + llvm::cl::desc( + "Whether to use the bareptr calling convention on the kernel " + "(warning this should be false until the GPU layering is fixed)"), + llvm::cl::init(false)}; + PassOptions::Option hostUseBarePtrCallConv{ + *this, "host-bare-ptr-calling-convention", + llvm::cl::desc( + "Whether to use the bareptr calling convention on the host (warning " + "this should be false until the GPU layering is fixed)"), + llvm::cl::init(false)}; +}; + +//===----------------------------------------------------------------------===// +// Building and Registering. +//===----------------------------------------------------------------------===// + +/// Adds the GPU to NVVM pipeline to the given pass manager. Transforms main +/// dialects into NVVM targets. Begins with GPU code regions, then handles host +/// code. +void buildLowerToNVVMPassPipeline(OpPassManager &pm, + const GPUToNVVMPipelineOptions &options); + +/// Register all pipeleines for the `gpu` dialect. void registerGPUToNVVMPipeline(); + } // namespace gpu } // namespace mlir diff --git a/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp b/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp index 5bee234e932a..0b4739214bf2 100644 --- a/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp +++ b/mlir/lib/Dialect/GPU/Pipelines/GPUToNVVMPipeline.cpp @@ -40,54 +40,14 @@ using namespace mlir; #if MLIR_CUDA_CONVERSIONS_ENABLED namespace { -struct GPUToNVVMPipelineOptions - : public PassPipelineOptions { - PassOptions::Option indexBitWidth{ - *this, "index-bitwidth", - llvm::cl::desc("Bitwidth of the index type for the host (warning this " - "should be 64 until the GPU layering is fixed)"), - llvm::cl::init(64)}; - PassOptions::Option cubinTriple{ - *this, "cubin-triple", - llvm::cl::desc("Triple to use to serialize to cubin."), - llvm::cl::init("nvptx64-nvidia-cuda")}; - PassOptions::Option cubinChip{ - *this, "cubin-chip", llvm::cl::desc("Chip to use to serialize to cubin."), - llvm::cl::init("sm_50")}; - PassOptions::Option cubinFeatures{ - *this, "cubin-features", - llvm::cl::desc("Features to use to serialize to cubin."), - llvm::cl::init("+ptx60")}; - PassOptions::Option cubinFormat{ - *this, "cubin-format", - llvm::cl::desc("Compilation format to use to serialize to cubin."), - llvm::cl::init("fatbin")}; - PassOptions::Option optLevel{ - *this, "opt-level", - llvm::cl::desc("Optimization level for NVVM compilation"), - llvm::cl::init(2)}; - PassOptions::Option kernelUseBarePtrCallConv{ - *this, "kernel-bare-ptr-calling-convention", - llvm::cl::desc( - "Whether to use the bareptr calling convention on the kernel " - "(warning this should be false until the GPU layering is fixed)"), - llvm::cl::init(false)}; - PassOptions::Option hostUseBarePtrCallConv{ - *this, "host-bare-ptr-calling-convention", - llvm::cl::desc( - "Whether to use the bareptr calling convention on the host (warning " - "this should be false until the GPU layering is fixed)"), - llvm::cl::init(false)}; -}; //===----------------------------------------------------------------------===// // Common pipeline //===----------------------------------------------------------------------===// -void buildCommonPassPipeline(OpPassManager &pm, - const GPUToNVVMPipelineOptions &options) { +void buildCommonPassPipeline( + OpPassManager &pm, const mlir::gpu::GPUToNVVMPipelineOptions &options) { pm.addPass(createConvertNVGPUToNVVMPass()); pm.addPass(createGpuKernelOutliningPass()); - pm.addPass(createConvertLinalgToLoopsPass()); pm.addPass(createConvertVectorToSCFPass()); pm.addPass(createConvertSCFToCFPass()); pm.addPass(createConvertNVVMToLLVMPass()); @@ -114,7 +74,7 @@ void buildCommonPassPipeline(OpPassManager &pm, // GPUModule-specific stuff. //===----------------------------------------------------------------------===// void buildGpuPassPipeline(OpPassManager &pm, - const GPUToNVVMPipelineOptions &options) { + const mlir::gpu::GPUToNVVMPipelineOptions &options) { pm.addNestedPass(createStripDebugInfoPass()); ConvertGpuOpsToNVVMOpsOptions opt; opt.useBarePtrCallConv = options.kernelUseBarePtrCallConv; @@ -129,7 +89,7 @@ void buildGpuPassPipeline(OpPassManager &pm, // Host Post-GPU pipeline //===----------------------------------------------------------------------===// void buildHostPostPipeline(OpPassManager &pm, - const GPUToNVVMPipelineOptions &options) { + const mlir::gpu::GPUToNVVMPipelineOptions &options) { GpuToLLVMConversionPassOptions opt; opt.hostBarePtrCallConv = options.hostUseBarePtrCallConv; opt.kernelBarePtrCallConv = options.kernelUseBarePtrCallConv; @@ -143,36 +103,28 @@ void buildHostPostPipeline(OpPassManager &pm, pm.addPass(createReconcileUnrealizedCastsPass()); } -void buildLowerToNVVMPassPipeline(OpPassManager &pm, - const GPUToNVVMPipelineOptions &options) { - //===----------------------------------------------------------------------===// - // Common pipeline - //===----------------------------------------------------------------------===// +} // namespace + +void mlir::gpu::buildLowerToNVVMPassPipeline( + OpPassManager &pm, const GPUToNVVMPipelineOptions &options) { + // Common pipelines buildCommonPassPipeline(pm, options); - //===----------------------------------------------------------------------===// - // GPUModule-specific stuff. - //===----------------------------------------------------------------------===// + // GPUModule-specific stuff buildGpuPassPipeline(pm, options); - //===----------------------------------------------------------------------===// - // Host post-GPUModule-specific stuff. - //===----------------------------------------------------------------------===// + // Host post-GPUModule-specific stuff buildHostPostPipeline(pm, options); } -} // namespace -namespace mlir { -namespace gpu { -void registerGPUToNVVMPipeline() { +void mlir::gpu::registerGPUToNVVMPipeline() { PassPipelineRegistration( - "gpu-lower-to-nvvm", - "The default pipeline lowers main dialects (arith, linalg, memref, scf, " + "gpu-lower-to-nvvm-pipeline", + "The default pipeline lowers main dialects (arith, memref, scf, " "vector, gpu, and nvgpu) to NVVM. It starts by lowering GPU code to the " "specified compilation target (default is fatbin) then lowers the host " "code.", buildLowerToNVVMPassPipeline); } -} // namespace gpu -} // namespace mlir + #endif // MLIR_CUDA_CONVERSIONS_ENABLED diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/dump-ptx.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/dump-ptx.mlir index 42348e39832a..0cc5d8645bb3 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/dump-ptx.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/dump-ptx.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm -debug-only=serialize-to-isa \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline -debug-only=serialize-to-isa \ // RUN: 2>&1 | FileCheck %s // CHECK: Generated by LLVM NVPTX Back-End diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-mma-2-4-f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-mma-2-4-f16.mlir index 62d0d9e1cac9..5a624e643429 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-mma-2-4-f16.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-mma-2-4-f16.mlir @@ -4,7 +4,7 @@ // RUN: mlir-opt \ // RUN: --pass-pipeline="builtin.module(gpu.module(strip-debuginfo,convert-gpu-to-nvvm,convert-nvgpu-to-nvvm,affine-expand-index-ops,lower-affine,convert-arith-to-llvm),convert-vector-to-llvm,canonicalize,cse)" \ // RUN: %s \ -// RUN: | mlir-opt --gpu-lower-to-nvvm="cubin-chip=sm_80 cubin-features=+ptx71 cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt --gpu-lower-to-nvvm-pipeline="cubin-chip=sm_80 cubin-features=+ptx71 cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_c_runner_utils \ diff --git a/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-reduction-distribute.mlir b/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-reduction-distribute.mlir index 94a57d7c2668..378e5b39415b 100644 --- a/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-reduction-distribute.mlir +++ b/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-reduction-distribute.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s -test-vector-warp-distribute="hoist-uniform distribute-transfer-write propagate-distribution" -canonicalize |\ // RUN: mlir-opt -test-vector-warp-distribute=rewrite-warp-ops-to-scf-if |\ // RUN: mlir-opt -lower-affine -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm \ -// RUN: -convert-arith-to-llvm -gpu-lower-to-nvvm | \ +// RUN: -convert-arith-to-llvm -gpu-lower-to-nvvm-pipeline | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_cuda_runtime \ // RUN: -shared-libs=%mlir_c_runner_utils \ diff --git a/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-warp-distribute.mlir b/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-warp-distribute.mlir index 896051ab5dd7..7e9234901ffa 100644 --- a/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-warp-distribute.mlir +++ b/mlir/test/Integration/Dialect/Vector/GPU/CUDA/test-warp-distribute.mlir @@ -2,7 +2,7 @@ // everything on the same thread. // RUN: mlir-opt %s -test-vector-warp-distribute=rewrite-warp-ops-to-scf-if -canonicalize | \ // RUN: mlir-opt -convert-vector-to-scf -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \ -// RUN: -gpu-lower-to-nvvm | \ +// RUN: -gpu-lower-to-nvvm-pipeline | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_cuda_runtime \ // RUN: -shared-libs=%mlir_c_runner_utils \ @@ -13,7 +13,7 @@ // RUN: mlir-opt %s -test-vector-warp-distribute="hoist-uniform distribute-transfer-write" \ // RUN: -test-vector-warp-distribute=rewrite-warp-ops-to-scf-if -canonicalize | \ // RUN: mlir-opt -convert-vector-to-scf -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \ -// RUN: -gpu-lower-to-nvvm | \ +// RUN: -gpu-lower-to-nvvm-pipeline | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_cuda_runtime \ // RUN: -shared-libs=%mlir_c_runner_utils \ @@ -23,7 +23,7 @@ // RUN: mlir-opt %s -test-vector-warp-distribute="hoist-uniform distribute-transfer-write propagate-distribution" \ // RUN: -test-vector-warp-distribute=rewrite-warp-ops-to-scf-if -canonicalize | \ // RUN: mlir-opt -convert-vector-to-scf -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \ -// RUN: -gpu-lower-to-nvvm | \ +// RUN: -gpu-lower-to-nvvm-pipeline | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_cuda_runtime \ // RUN: -shared-libs=%mlir_c_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f16-f16-accum.mlir b/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f16-f16-accum.mlir index d4bd51aab035..8379710ebbbb 100644 --- a/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f16-f16-accum.mlir +++ b/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f16-f16-accum.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s \ // RUN: -transform-interpreter \ // RUN: -test-transform-dialect-erase-schedule \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_80 cubin-features=+ptx76 cubin-format=%gpu_compilation_format" \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_80 cubin-features=+ptx76 cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f32.mlir b/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f32.mlir index 3e5f291db8e7..afed0ef667a2 100644 --- a/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f32.mlir +++ b/mlir/test/Integration/GPU/CUDA/TensorCore/sm80/transform-mma-sync-matmul-f32.mlir @@ -11,7 +11,7 @@ // RUN: mlir-opt %s \ // RUN: -transform-interpreter \ // RUN: -test-transform-dialect-erase-schedule \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_80 cubin-features=+ptx76 cubin-format=%gpu_compilation_format" \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_80 cubin-features=+ptx76 cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f16.mlir b/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f16.mlir index bbeddd5bb228..958da79ee166 100644 --- a/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f16.mlir +++ b/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f16.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-chip=sm_70 cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_70 cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32-bare-ptr.mlir b/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32-bare-ptr.mlir index d5950eae2543..6b5b635c8534 100644 --- a/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32-bare-ptr.mlir +++ b/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32-bare-ptr.mlir @@ -3,7 +3,7 @@ // Similar to the wmma-matmul-f32 but but with the memref bare pointer lowering convention. // This test also uses gpu.memcpy operations (instead of gpu.host_register). // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="host-bare-ptr-calling-convention=1 kernel-bare-ptr-calling-convention=1 cubin-chip=sm_70 cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="host-bare-ptr-calling-convention=1 kernel-bare-ptr-calling-convention=1 cubin-chip=sm_70 cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --entry-point-result=void \ diff --git a/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32.mlir b/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32.mlir index c75f9c1b5649..7fbe3e1c8819 100644 --- a/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32.mlir +++ b/mlir/test/Integration/GPU/CUDA/TensorCore/wmma-matmul-f32.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-chip=sm_70 cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_70 cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/all-reduce-and.mlir b/mlir/test/Integration/GPU/CUDA/all-reduce-and.mlir index fe999e0aa575..9e10aab0f381 100644 --- a/mlir/test/Integration/GPU/CUDA/all-reduce-and.mlir +++ b/mlir/test/Integration/GPU/CUDA/all-reduce-and.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ @@ -8,7 +8,7 @@ // Same as above but with the memref bare pointer lowering convention. // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="kernel-bare-ptr-calling-convention=1 cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="kernel-bare-ptr-calling-convention=1 cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/all-reduce-maxsi.mlir b/mlir/test/Integration/GPU/CUDA/all-reduce-maxsi.mlir index dcd503c7bd80..c2ea7919cc3f 100644 --- a/mlir/test/Integration/GPU/CUDA/all-reduce-maxsi.mlir +++ b/mlir/test/Integration/GPU/CUDA/all-reduce-maxsi.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/all-reduce-minsi.mlir b/mlir/test/Integration/GPU/CUDA/all-reduce-minsi.mlir index 8236550feb11..db649cbeb194 100644 --- a/mlir/test/Integration/GPU/CUDA/all-reduce-minsi.mlir +++ b/mlir/test/Integration/GPU/CUDA/all-reduce-minsi.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/all-reduce-op.mlir b/mlir/test/Integration/GPU/CUDA/all-reduce-op.mlir index 6f965c225e2d..60323cee952a 100644 --- a/mlir/test/Integration/GPU/CUDA/all-reduce-op.mlir +++ b/mlir/test/Integration/GPU/CUDA/all-reduce-op.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/all-reduce-or.mlir b/mlir/test/Integration/GPU/CUDA/all-reduce-or.mlir index 340db39f5d28..1501160e98a1 100644 --- a/mlir/test/Integration/GPU/CUDA/all-reduce-or.mlir +++ b/mlir/test/Integration/GPU/CUDA/all-reduce-or.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/all-reduce-region.mlir b/mlir/test/Integration/GPU/CUDA/all-reduce-region.mlir index b4fc32ff9b83..8e683f360f10 100644 --- a/mlir/test/Integration/GPU/CUDA/all-reduce-region.mlir +++ b/mlir/test/Integration/GPU/CUDA/all-reduce-region.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/all-reduce-xor.mlir b/mlir/test/Integration/GPU/CUDA/all-reduce-xor.mlir index f43a095584d6..b1cae5b3f971 100644 --- a/mlir/test/Integration/GPU/CUDA/all-reduce-xor.mlir +++ b/mlir/test/Integration/GPU/CUDA/all-reduce-xor.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir b/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir index 7f5b38b34c89..41024a003b18 100644 --- a/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir +++ b/mlir/test/Integration/GPU/CUDA/gpu-to-cubin.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/multiple-all-reduce.mlir b/mlir/test/Integration/GPU/CUDA/multiple-all-reduce.mlir index a894030d4308..512f4902e5ec 100644 --- a/mlir/test/Integration/GPU/CUDA/multiple-all-reduce.mlir +++ b/mlir/test/Integration/GPU/CUDA/multiple-all-reduce.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/printf.mlir b/mlir/test/Integration/GPU/CUDA/printf.mlir index 9555a77f45f1..99ea1208e9c5 100644 --- a/mlir/test/Integration/GPU/CUDA/printf.mlir +++ b/mlir/test/Integration/GPU/CUDA/printf.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/shuffle.mlir b/mlir/test/Integration/GPU/CUDA/shuffle.mlir index 4e5bb3e8f5ca..cd11592c2dce 100644 --- a/mlir/test/Integration/GPU/CUDA/shuffle.mlir +++ b/mlir/test/Integration/GPU/CUDA/shuffle.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/sm90/asd b/mlir/test/Integration/GPU/CUDA/sm90/asd new file mode 100644 index 000000000000..353d8e7c16b7 --- /dev/null +++ b/mlir/test/Integration/GPU/CUDA/sm90/asd @@ -0,0 +1,207 @@ +module attributes {gpu.container_module} { + llvm.mlir.global private constant @vector_print_str_0(dense<[73, 110, 99, 111, 114, 114, 101, 99, 116, 32, 82, 101, 115, 117, 108, 116, 115, 32, 58, 10, 0]> : tensor<21xi8>) {addr_space = 0 : i32} : !llvm.array<21 x i8> + llvm.func @printNewline() + llvm.func @printI64(i64) + llvm.func @printString(!llvm.ptr) + llvm.mlir.global private constant @vector_print_str(dense<[67, 111, 114, 114, 101, 99, 116, 32, 82, 101, 115, 117, 108, 116, 115, 32, 58, 10, 0]> : tensor<19xi8>) {addr_space = 0 : i32} : !llvm.array<19 x i8> + llvm.func @malloc(i64) -> !llvm.ptr + llvm.mlir.global private @__mbarrier() {addr_space = 3 : i32, alignment = 8 : i64} : !llvm.array<2 x i64> + llvm.func @printMemrefF32(i64, !llvm.ptr) attributes {sym_visibility = "private"} + llvm.mlir.global private @dynamicShmem() {addr_space = 3 : i32, alignment = 16 : i64} : !llvm.array<0 x f16> + llvm.mlir.global private @accShmem() {addr_space = 3 : i32, alignment = 16 : i64} : !llvm.array<0 x f32> + llvm.func @main() { + %0 = llvm.mlir.constant(2 : index) : i64 + %1 = llvm.mlir.constant(0 : i8) : i8 + %2 = llvm.mlir.constant(64 : index) : i64 + %3 = llvm.mlir.constant(65536 : i32) : i32 + %4 = llvm.mlir.constant(16 : index) : i64 + %5 = llvm.mlir.constant(8 : index) : i64 + %6 = llvm.mlir.constant(0.000000e+00 : f32) : f32 + %7 = llvm.mlir.constant(6 : i32) : i64 + %8 = llvm.mlir.constant(5 : i32) : i64 + %9 = llvm.mlir.constant(0 : i32) : i64 + %10 = llvm.mlir.constant(3 : i32) : i64 + %11 = llvm.mlir.constant(1 : i32) : i32 + %12 = llvm.mlir.constant(0 : i32) : i32 + %13 = llvm.mlir.constant(9.99999993E-9 : f32) : f32 + %14 = llvm.mlir.constant(1 : index) : i64 + %15 = llvm.mlir.constant(0 : index) : i64 + %16 = llvm.mlir.constant(128 : index) : i64 + %17 = llvm.mlir.zero : !llvm.ptr + %18 = llvm.getelementptr %17[16384] : (!llvm.ptr) -> !llvm.ptr, f16 + %19 = llvm.ptrtoint %18 : !llvm.ptr to i64 + %20 = llvm.call @malloc(%19) : (i64) -> !llvm.ptr + %21 = llvm.call @malloc(%19) : (i64) -> !llvm.ptr + %22 = llvm.getelementptr %17[16384] : (!llvm.ptr) -> !llvm.ptr, f32 + %23 = llvm.ptrtoint %22 : !llvm.ptr to i64 + %24 = llvm.call @malloc(%23) : (i64) -> !llvm.ptr + %25 = llvm.call @malloc(%23) : (i64) -> !llvm.ptr + llvm.br ^bb1(%15 : i64) + ^bb1(%26: i64): // 2 preds: ^bb0, ^bb5 + %27 = llvm.icmp "slt" %26, %16 : i64 + llvm.cond_br %27, ^bb2, ^bb6 + ^bb2: // pred: ^bb1 + llvm.br ^bb3(%15 : i64) + ^bb3(%28: i64): // 2 preds: ^bb2, ^bb4 + %29 = llvm.icmp "slt" %28, %16 : i64 + llvm.cond_br %29, ^bb4, ^bb5 + ^bb4: // pred: ^bb3 + %30 = llvm.mul %26, %16 : i64 + %31 = llvm.add %30, %28 : i64 + %32 = llvm.udiv %31, %5 : i64 + %33 = llvm.urem %32, %4 : i64 + %34 = llvm.trunc %33 : i64 to i32 + %35 = llvm.sitofp %34 : i32 to f16 + %36 = llvm.getelementptr %21[%31] : (!llvm.ptr, i64) -> !llvm.ptr, f16 + llvm.store %35, %36 : f16, !llvm.ptr + %37 = llvm.mul %28, %2 : i64 + %38 = llvm.add %37, %26 : i64 + %39 = llvm.udiv %38, %5 : i64 + %40 = llvm.urem %39, %4 : i64 + %41 = llvm.trunc %40 : i64 to i32 + %42 = llvm.sitofp %41 : i32 to f16 + %43 = llvm.mul %28, %16 : i64 + %44 = llvm.add %43, %26 : i64 + %45 = llvm.getelementptr %20[%44] : (!llvm.ptr, i64) -> !llvm.ptr, f16 + llvm.store %42, %45 : f16, !llvm.ptr + %46 = llvm.getelementptr %24[%31] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + llvm.store %6, %46 : f32, !llvm.ptr + %47 = llvm.getelementptr %25[%31] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + llvm.store %6, %47 : f32, !llvm.ptr + %48 = llvm.add %28, %14 : i64 + llvm.br ^bb3(%48 : i64) + ^bb5: // pred: ^bb3 + %49 = llvm.add %26, %14 : i64 + llvm.br ^bb1(%49 : i64) + ^bb6: // pred: ^bb1 + %50 = llvm.call @mgpuStreamCreate() : () -> !llvm.ptr + %51 = llvm.call @mgpuMemAlloc(%19, %50, %1) : (i64, !llvm.ptr, i8) -> !llvm.ptr + %52 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %53 = llvm.insertvalue %51, %52[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %54 = llvm.insertvalue %51, %53[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %55 = llvm.insertvalue %15, %54[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %56 = llvm.insertvalue %16, %55[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %57 = llvm.insertvalue %16, %56[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %58 = llvm.insertvalue %16, %57[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %59 = llvm.insertvalue %14, %58[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %60 = llvm.call @mgpuMemAlloc(%19, %50, %1) : (i64, !llvm.ptr, i8) -> !llvm.ptr + %61 = llvm.insertvalue %60, %52[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %62 = llvm.insertvalue %60, %61[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %63 = llvm.insertvalue %15, %62[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %64 = llvm.insertvalue %16, %63[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %65 = llvm.insertvalue %16, %64[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %66 = llvm.insertvalue %16, %65[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %67 = llvm.insertvalue %14, %66[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %68 = llvm.call @mgpuMemAlloc(%23, %50, %1) : (i64, !llvm.ptr, i8) -> !llvm.ptr + llvm.call @mgpuMemcpy(%51, %20, %19, %50) : (!llvm.ptr, !llvm.ptr, i64, !llvm.ptr) -> () + llvm.call @mgpuMemcpy(%60, %21, %19, %50) : (!llvm.ptr, !llvm.ptr, i64, !llvm.ptr) -> () + %69 = llvm.alloca %14 x !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> : (i64) -> !llvm.ptr + llvm.store %59, %69 : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>, !llvm.ptr + %70 = llvm.alloca %14 x !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> : (i64) -> !llvm.ptr + llvm.store %67, %70 : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>, !llvm.ptr + %71 = llvm.alloca %8 x i64 : (i64) -> !llvm.ptr + llvm.store %16, %71 : i64, !llvm.ptr + %72 = llvm.getelementptr %71[1] : (!llvm.ptr) -> !llvm.ptr, !llvm.ptr + llvm.store %2, %72 : i64, !llvm.ptr + %73 = llvm.call @mgpuTensorMapEncodeTiledMemref(%0, %69, %7, %9, %10, %9, %9, %71) : (i64, !llvm.ptr, i64, i64, i64, i64, i64, !llvm.ptr) -> !llvm.ptr + %74 = llvm.alloca %8 x i64 : (i64) -> !llvm.ptr + llvm.store %2, %74 : i64, !llvm.ptr + %75 = llvm.getelementptr %74[1] : (!llvm.ptr) -> !llvm.ptr, !llvm.ptr + llvm.store %2, %75 : i64, !llvm.ptr + %76 = llvm.call @mgpuTensorMapEncodeTiledMemref(%0, %70, %7, %9, %10, %9, %9, %74) : (i64, !llvm.ptr, i64, i64, i64, i64, i64, !llvm.ptr) -> !llvm.ptr + gpu.launch_func @main_kernel::@main_kernel blocks in (%14, %14, %14) threads in (%16, %14, %14) : i64 dynamic_shared_memory_size %3 args(%68 : !llvm.ptr, %68 : !llvm.ptr, %15 : i64, %16 : i64, %16 : i64, %16 : i64, %14 : i64, %73 : !llvm.ptr, %76 : !llvm.ptr) + llvm.call @mgpuMemcpy(%24, %68, %23, %50) : (!llvm.ptr, !llvm.ptr, i64, !llvm.ptr) -> () + llvm.br ^bb7(%15 : i64) + ^bb7(%77: i64): // 2 preds: ^bb6, ^bb14 + %78 = llvm.icmp "slt" %77, %16 : i64 + llvm.cond_br %78, ^bb8, ^bb15 + ^bb8: // pred: ^bb7 + llvm.br ^bb9(%15 : i64) + ^bb9(%79: i64): // 2 preds: ^bb8, ^bb13 + %80 = llvm.icmp "slt" %79, %16 : i64 + llvm.cond_br %80, ^bb10, ^bb14 + ^bb10: // pred: ^bb9 + llvm.br ^bb11(%15 : i64) + ^bb11(%81: i64): // 2 preds: ^bb10, ^bb12 + %82 = llvm.icmp "slt" %81, %16 : i64 + llvm.cond_br %82, ^bb12, ^bb13 + ^bb12: // pred: ^bb11 + %83 = llvm.mul %77, %16 : i64 + %84 = llvm.add %83, %81 : i64 + %85 = llvm.getelementptr %20[%84] : (!llvm.ptr, i64) -> !llvm.ptr, f16 + %86 = llvm.load %85 : !llvm.ptr -> f16 + %87 = llvm.mul %81, %16 : i64 + %88 = llvm.add %87, %79 : i64 + %89 = llvm.getelementptr %21[%88] : (!llvm.ptr, i64) -> !llvm.ptr, f16 + %90 = llvm.load %89 : !llvm.ptr -> f16 + %91 = llvm.add %83, %79 : i64 + %92 = llvm.getelementptr %25[%91] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + %93 = llvm.load %92 : !llvm.ptr -> f32 + %94 = llvm.fpext %86 : f16 to f32 + %95 = llvm.fpext %90 : f16 to f32 + %96 = llvm.fmul %94, %95 : f32 + %97 = llvm.fadd %93, %96 : f32 + llvm.store %97, %92 : f32, !llvm.ptr + %98 = llvm.add %81, %14 : i64 + llvm.br ^bb11(%98 : i64) + ^bb13: // pred: ^bb11 + %99 = llvm.add %79, %14 : i64 + llvm.br ^bb9(%99 : i64) + ^bb14: // pred: ^bb9 + %100 = llvm.add %77, %14 : i64 + llvm.br ^bb7(%100 : i64) + ^bb15: // pred: ^bb7 + llvm.br ^bb16(%15, %12, %12 : i64, i32, i32) + ^bb16(%101: i64, %102: i32, %103: i32): // 2 preds: ^bb15, ^bb24 + %104 = llvm.icmp "slt" %101, %16 : i64 + llvm.cond_br %104, ^bb17, ^bb25 + ^bb17: // pred: ^bb16 + llvm.br ^bb18(%15, %102, %103 : i64, i32, i32) + ^bb18(%105: i64, %106: i32, %107: i32): // 2 preds: ^bb17, ^bb23 + %108 = llvm.icmp "slt" %105, %16 : i64 + llvm.cond_br %108, ^bb19, ^bb24 + ^bb19: // pred: ^bb18 + %109 = llvm.mul %101, %16 : i64 + %110 = llvm.add %109, %105 : i64 + %111 = llvm.getelementptr %25[%110] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + %112 = llvm.load %111 : !llvm.ptr -> f32 + %113 = llvm.getelementptr %24[%110] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + %114 = llvm.load %113 : !llvm.ptr -> f32 + %115 = llvm.fsub %112, %114 : f32 + %116 = llvm.intr.fabs(%115) : (f32) -> f32 + %117 = llvm.fcmp "ult" %13, %116 : f32 + llvm.cond_br %117, ^bb20, ^bb21 + ^bb20: // pred: ^bb19 + %118 = llvm.add %106, %11 : i32 + llvm.br ^bb22(%118, %107 : i32, i32) + ^bb21: // pred: ^bb19 + %119 = llvm.add %107, %11 : i32 + llvm.br ^bb22(%106, %119 : i32, i32) + ^bb22(%120: i32, %121: i32): // 2 preds: ^bb20, ^bb21 + llvm.br ^bb23 + ^bb23: // pred: ^bb22 + %122 = llvm.add %105, %14 : i64 + llvm.br ^bb18(%122, %120, %121 : i64, i32, i32) + ^bb24: // pred: ^bb18 + %123 = llvm.add %101, %14 : i64 + llvm.br ^bb16(%123, %106, %107 : i64, i32, i32) + ^bb25: // pred: ^bb16 + %124 = llvm.mlir.addressof @vector_print_str : !llvm.ptr + llvm.call @printString(%124) : (!llvm.ptr) -> () + %125 = llvm.sext %103 : i32 to i64 + llvm.call @printI64(%125) : (i64) -> () + llvm.call @printNewline() : () -> () + %126 = llvm.mlir.addressof @vector_print_str_0 : !llvm.ptr + llvm.call @printString(%126) : (!llvm.ptr) -> () + %127 = llvm.sext %102 : i32 to i64 + llvm.call @printI64(%127) : (i64) -> () + llvm.call @printNewline() : () -> () + llvm.return + } + gpu.binary @main_kernel [#gpu.object<#nvvm.target, "P\EDU\BA\01\00\10\00\A83\00\00\00\00\00\00\02\00\01\01@\00\00\00p$\00\00\00\00\00\00\00\00\00\00\00\00\00\00\07\00\01\00Z\00\00\00\00\00\00\00\00\00\00\00\11\00\10\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\7FELF\02\01\013\07\00\00\00\00\00\00\00\02\00\BE\00{\00\00\00\00\00\00\00\00\00\00\00X#\00\00\00\00\00\00X \00\00\00\00\00\00Z\0DZ\00@\008\00\05\00@\00\0C\00\01\00\00.shstrtab\00.strtab\00.symtab\00.symtab_shndx\00.nv.uft.entry\00.nv.info\00.text.main_kernel\00.nv.info.main_kernel\00.nv.shared.main_kernel\00.rel.text.main_kernel\00.rela.text.main_kernel\00.debug_frame\00.rel.debug_frame\00.rela.debug_frame\00.nv.constant0.main_kernel\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00.shstrtab\00.strtab\00.symtab\00.symtab_shndx\00.nv.uft.entry\00.nv.info\00main_kernel\00.text.main_kernel\00.nv.info.main_kernel\00.nv.shared.main_kernel\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00.rel.text.main_kernel\00.rela.text.main_kernel\00$__dynamicShmem__31\00$____mbarrier__33\00$__accShmem__35\00.debug_frame\00.rel.debug_frame\00.rela.debug_frame\00.nv.constant0.main_kernel\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00L\00\00\00\03\00\09\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00s\00\00\00\03\00\0A\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02\01\00\00\03\00\04\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00@\00\00\00\12\10\09\00\00\00\00\00\00\00\00\00\80\18\00\00\00\00\00\002\01\00\00\03\00\0B\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\FF\FF\FF\FF$\00\00\00\00\00\00\00\FF\FF\FF\FF\FF\FF\FF\FF\03\00\04|\FF\FF\FF\FF\0F\0C\81\80\80(\00\08\FF\81\80(\08\81\80\80(\00\00\00\FF\FF\FF\FF,\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\80\18\00\00\00\00\00\00\04T\00\00\00\0C\81\80\80(\00\04\A0\05\00\00\00\00\00\00\04/\08\00\05\00\00\00\9A\00\00\00\04#\08\00\05\00\00\00\00\00\00\00\04\12\08\00\05\00\00\00\00\00\00\00\04\11\08\00\05\00\00\00\00\00\00\00\047\04\00{\00\00\00\04\17\0C\00\00\00\00\00\08\00@\00\00\F0!\00\04\17\0C\00\00\00\00\00\07\008\00\00\F0!\00\04\17\0C\00\00\00\00\00\06\000\00\00\F0!\00\04\17\0C\00\00\00\00\00\05\00(\00\00\F0!\00\04\17\0C\00\00\00\00\00\04\00 \00\00\F0!\00\04\17\0C\00\00\00\00\00\03\00\18\00\00\F0!\00\04\17\0C\00\00\00\00\00\02\00\10\00\00\F0!\00\04\17\0C\00\00\00\00\00\01\00\08\00\00\F0!\00\04\17\0C\00\00\00\00\00\00\00\00\00\00\F0!\00\03\1B\FF\00\0490\00\C0\00\00\00\FF\00\00\00\00\00\00\00\00\01\09\00\00\01\00\00\FF\00\00\00\08\00\00\00\00\01\09\00p\09\00\00\00\00\00\00\00\00\00\00\0A\01?\00\038\02\00\04\1C\0C\00@\12\00\00\80\17\00\00\D0\17\00\00\04\1E\04\00\00\00\00\00\03\19H\00\04\0A\08\00\06\00\00\00\10\02H\00\00\00\00\00D\00\00\00\00\00\00\00\02\00\00\00\05\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\82{\01\FF\00\0A\00\00\00\08\00\00\00$\0E\00\C3y\04\00\00\00\00\00\00\88\00\00\00b\0E\00\19y\06\00\00\00\00\00\00!\00\00\00\A2\0E\00\82x\08\00\00\04\00\00\00\00\00\00\00\E2\0F\00\82x\0A\00\FE\FF\1F\00\00\00\00\00\00\E2\0F\00\90x\09\08\08\00\00\00?\E0\FF\0F\00\E2\0F\00Ey\00\00\A0\04\00\00\00\00\80\03\00\E2\0F\00\82x\0B\00\00\F8\FF\7F\00\00\00\00\00\E2\0F\00\B9z\06\00\00\92\00\00\00\0A\00\00\00\E2\0F\00\B9z\0C\00\00\94\00\00\00\0A\00\00\00\E2\0F\00\96x\09\04T\06\00\00\09\00\00\08\00\E4/\00\96x\08\04T\06\00\00\08\00\00\08\00\E2\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\E2\0F\00\0Cr\00\06\FF\00\00\00pR\F0\03\00\E2O\00\C6s\00\00\00\00\00\00\00\00\00\00\00n\0E\00\B2u?\09\0A\00\00\00\00\01\00\08\00b\02\00\18y\00\00\00\00\00\00\00\00\00\00\00\E2\0F\00\B2u?\09\0A\08\00\00\00\01\00\08\00b\02\00\B9y\00\06\00\00\00\00\00\00\04\08\00\E2\03\00\B9y\00\0C\00\00\00\00\00\00\04\08\00\E4\03\00G\09\EC\00\00\00\00\00\00\00\80\03\00\EA/\00$t\00\FF\00\80\00\00\FF\00\8E\07\00\E2\0F\00\1Cx\00\00\00\00\00\00p\F0\F0\03\00\E2\0F\00\82|\0A\00?\00\00\00\00\00\00\08\00\E4\0F\00\A7y\FF\FF\00\00\00\00\09\00\00\08\00\F4\03\00/\08?\00\00\00\00\00\00\00\82\03\00\E2\0F\00\82|\0B\00\0A\00\00\00\00\00\00\08\00\E4\0F\00\B4u\00\06\08\00\00\00\00\80\00\08\00\F4\05\00\1C\18\00\00\00\00\00\00p\E1\F0\00\00\C4\0F\00\1Cx\00\00\00\00\00\00p\E1\F2\03\00\D6\0F\00G\09\E8\00\FD\FF\FF\FF\FF\FF\93\03\00\EAO\00\1Cx\00\00\00\00\00\00p\F0\F0\03\00\E2\0F\00\90x\10\08\00\80\00\00?\E0\FF\0F\00\D8\0F\00/\08?\00\00\00\00\00\00\00\82\03\00\E2\0F\00\82|\11\00\09\00\00\00\00\00\00\08\00\E2\0F\00\82|\12\00\0A\00\00\00\00\00\00\08\00\E2\0F\00\82|\13\00\0A\00\00\00\00\00\00\08\00\E4\0F\00\B4u\00\0C\10\00\00\00\00\80\00\08\00\F0\05\00\1C\18\00\00\00\00\00\00p\E1\F0\00\00\C4\0F\00\1Cx\00\00\00\00\00\00p\E1\F2\03\00\D6\0F\00G\09\E0\00\FD\FF\FF\FF\FF\FF\93\03\00\EAO\00\1Cx\00\00\00\00\00\00p\F0\F0\03\00\E2\0F\00\90x\10\08\00\A0\00\00?\E0\FF\0F\00\E2\0F\00\82x\12\00@\00\00\00\00\00\00\00\00\D6\0F\00/\08?\00\00\00\00\00\00\00\82\03\00\E2\0F\00\82|\11\00\09\00\00\00\00\00\00\08\00\E2\0F\00\82|\13\00\0A\00\00\00\00\00\00\08\00\E4\0F\00\B4u\00\0C\10\00\00\00\00\80\00\08\00\F2\05\00\1C\18\00\00\00\00\00\00p\E1\F0\00\00\C4\0F\00\1Cx\00\00\00\00\00\00p\E1\F2\03\00\D6\0F\00G\09\E4\00\FD\FF\FF\FF\FF\FF\93\03\00\EAO\00\A7y\FF\FF\00\08\00\00\09\00\00\08\00\E2\05\00\1Cx\00\00\00\00\00\00p\F0\F0\03\00\E2\0F\00\90x\10\08\00@\00\00?\E0\FF\0F\00\E4\0F\00\90x\11\09\08\00\00\00?\E0\FF\0F\00\D8\0F\00/\08?\00\00\00\00\00\00\00\82\03\00\E2\0F\00\82x\12\00@\00\00\00\00\00\00\00\00\E2\0F\00\82|\13\00\0A\00\00\00\00\00\00\08\00\E4\0F\00\B4u\00\06\10\00\00\00\00\80\00\08\00\F2\07\00\1C\18\00\00\00\00\00\00p\E1\F0\00\00\C4\0F\00\1Cx\00\00\00\00\00\00p\E1\F2\03\00\D6\0F\00G\09\E4\00\FD\FF\FF\FF\FF\FF\93\03\00\EA\8F\00\1Cx\00\00\00\00\00\00p\F0\F0\03\00\E2\0F\00\90x\10\08\00\C0\00\00?\E0\FF\0F\00\D8\0F\00/\08?\00\00\00\00\00\00\00\82\03\00\E2\0F\00\82|\12\00\0A\00\00\00\00\00\00\08\00\E2\0F\00\82x\13\00@\00\00\00\00\00\00\00\00\E4\0F\00\B4u\00\0C\10\00\00\00\00\80\00\08\00\F2\07\00\1C\18\00\00\00\00\00\00p\E1\F0\00\00\C4\0F\00\1Cx\00\00\00\00\00\00p\E1\F2\03\00\D6\0F\00G\09\E4\00\FD\FF\FF\FF\FF\FF\93\03\00\EA\8F\00\1Cx\00\00\00\00\00\00p\F0\F0\03\00\E2\0F\00\90x\10\08\00\E0\00\00?\E0\FF\0F\00\D8\0F\00/\08?\00\00\00\00\00\00\00\82\03\00\E2\0F\00\82x\12\00@\00\00\00\00\00\00\00\00\E2\0F\00\82x\13\00@\00\00\00\00\00\00\00\00\E4\0F\00\B4u\00\0C\10\00\00\00\00\80\00\08\00\F2\07\00\1C\18\00\00\00\00\00\00p\E1\F0\00\00\C4\0F\00\1Cx\00\00\00\00\00\00p\E1\F2\03\00\D6\0F\00G\09\E4\00\FD\FF\FF\FF\FF\FF\93\03\00\EA\8F\00Ay\00\00\00\00\00\00\00\00\80\03\00\EA\0F\00\05xX\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xZ\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\\\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x^\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x`\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xb\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05xd\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xf\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xh\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xj\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xl\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xn\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05xp\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xr\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xt\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xv\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xx\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xz\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05x|\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x~\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\80\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\82\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\84\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\86\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05x\88\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\8A\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\8C\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\8E\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\90\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\92\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05x\94\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\96\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\18\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\1A\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\1C\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\1E\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05x \00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x\22\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x$\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x&\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x(\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x*\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05x,\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x.\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x0\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x2\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x4\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x6\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05x8\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x:\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x<\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x>\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05x@\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xB\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05xD\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xF\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xH\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xJ\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xL\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xN\00\00\00\00\00\00\FF\01\00\00\C4\0F\00\05xP\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xR\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xT\00\00\00\00\00\00\FF\01\00\00\E4\0F\00\05xV\00\00\00\00\00\00\FF\01\00\00\E2\0F\00\9Cx\00\00\00\00\00\00p\F0\F0\03\00\E2\0F\00\82|\05\00?\00\00\00\00\00\00\08\00\E2\0F\00\82|\04\00?\00\00\00\00\00\00\08\00\D2\0F\00\1Cx\00\00\00\00\00\00\08\F0\F0\03\00\E2\0F\00\91r\06\05\09\00\00\00?\18\8E\0F\00\D8\0F\00$~\00\FF\06\00\00\00\FF\00\8E\0F\00\C8o\00\A7u\00\00\FF\00\00\00\7F\01\02\08\00b\02\00\1Ay\00\00\00\90\00\00\00\00\00\00\00\C8\0F\00]\99\00\00\81\96\98\00\00\00\90\03\00\EA\0F\00\A7\95\00\00\FF\00\00\00\7F\00\02\08\00$\0E\00G\99\E8\00\FC\FF\FF\FF\FF\FF\83\03\00\EA\1F\00\91r\07\05\08\00\00\00?p\80\0F\00\E2\0F\00\C5y\00\00\00\00\00\00\00\00\00\00\00\E2\0F\00\82x\19\00@\00\00@\00\00\00\00\00\E2\0F\00\82x\1B\00@\00\00@\00\00\00\00\00\E2\0F\00\91r\0A\05?\00\00\00\04t\0F\08\00\E4\0F\00\90x\05\07\00\80\00\00?\E0\F1\0F\00\E4\0F\00\99x\07\07\04\00\00\00\0A\12\00\08\00\E4\0F\00\90r\06?\0A\00\00\00?\E4\7F\08\00\C8\0F\00\99x\05\05\04\00\00\00\06\12\00\08\00\E4\0F\00\92x\06\07\FF?\00\00?\C0\8E\0F\00\E4\0F\00\92x\05\05\FF?\00\00?\C0\8E\0F\00\E4\0F\00\92x\18\06\00\00\00\04?\FC\8E\0F\00\E4\0F\00\92x\1A\05\00\00\00\02?\FC\8E\0F\00\E4\0F\00\90x\14\06\02\00\00\04?\E0\F1\0F\00\C4\0F\00\90x\16\05\80\00\00\02?\E0\F3\0F\00\E4\0F\00\90x\15?@\00\00@?\E4\7F\08\00\E4\0F\00\90x\17?@\00\00@?\E4\FF\08\00\E2\0F\00\F0yX\18\00\00\E0\01X\08p\08\00\E2\0F\00\90x\10\06\04\00\00\04?\E0\F1\0F\00\E4\0F\00\90x\12\05\00\01\00\02?\E0\F3\0F\00\E4\0F\00\90x\11?@\00\00@?\E4\7F\08\00\C4\0F\00\90x\13?@\00\00@?\E4\FF\08\00\E4\0F\00\90x\0C\06\06\00\00\04?\E0\F1\0F\00\E4\0F\00\90x\0E\05\80\01\00\02?\E0\F3\0F\00\E4\0F\00\90x\0D?@\00\00@?\E4\7F\08\00\E4\0F\00\90x\18\06\00\02\00\04?\E0\F1\0F\00\E4\0F\00\90x\0F?@\00\00@?\E4\FF\08\00\C4\0F\00\90x\19?@\00\00@?\E4\7F\08\00\E2\0F\00\82x\05\00\01\00\00\00\00\00\00\00\00\E2\0F\00\F0yX\14\00\00\E0\01X\08p\08\00\E2\0F\00\90x\14\06\02\02\00\04?\E0\F1\0F\00\C8\0F\00\90x\15?@\00\00@?\E4\7F\08\00\CE\0F\00\F0yX\10\00\00\E0\01X\08p\08\00\E2\0F\00\90x\10\06\04\02\00\04?\E0\F1\0F\00\C8\0F\00\90x\11?@\00\00@?\E4\7F\08\00\CE\0F\00\F0yX\0C\00\00\E0\01X\08p\08\00\E2\0F\00\90x\0C\06\06\02\00\04?\E0\F1\0F\00\C8\0F\00\90x\0D?@\00\00@?\E4\7F\08\00\E4\0F\00\9Cx\00\00\00\00\00\00p\E8\F0\03\00\CA\0F\00\F0y\18\18\00\00\E0\01\18\08p\08\00\D8\0F\00\F0y\18\14\00\00\E0\01\18\08p\08\00\D8\0F\00\F0y\18\10\00\00\E0\01\18\08p\08\00\D8\0F\00\F0y\18\0C\00\00\E0\01\18\08\00\08\00\E6\0F\00\C5y\00\00\00\80\00\00\00\01\01\00\00\E4\0F\00G\09,\00\FC\FF\FF\FF\FF\FF\83\03\00\EA\0F\00\C3y\07\00\00\00\00\00\00\88\00\00\00b\0E\00\19x\00\FF\02\00\00\00\06\16\01\00\00\E2\0F\10$x\02\06\02\00\00\00\FF\00\8E\07\00\E2\0F\00\19x\03\FF\01\00\00\00\06\16\01\00\00\E2\0F\00\82x\06\00\00\04\00\00\00\00\00\00\00\E2\0F\00\1Ax\00\00\03\00\00\00\00\00\00\00\00\E2\0F\00\90x\06\06 \00\00\00?\E0\FF\0F\00\E2\0F\00\12x\02\02\06\00\00\00\FF\C0\8E\07\00\E2\0F\00\C5y\00\00\00\80\00\00\00\00\01\00\00\E4\0F\00\12x\00\00\F0\FF\FF\7F\03\F8\8E\07\00\C4\0F\00\0Cx\00\06\FF\0F\00\00p@\F0\03\00\E4\0F\00\12x\04\00\08\00\00\00\FF\FC\8E\07\00\E2\0F\04$x\03\00\80\00\00\00\FF\00\8E\07\00\E4\0F\006x\05\00@\00\00\00\00\00\00\00\00\E4\0F\006x\00\00H\00\00\00\00\00\00\00\00\E2\0F\00\12r\08\03\02\00\00\00\FF\FC\8E\07\00\E2\0F\08$x\03\04\80\00\00\00\FF\00\8E\07\00\E4\0F\00$x\05\05\80\00\00\00\FF\00\8E\07\00\E2\0F\00\96x\06\07T\06\00\00\06\00\00\08\00\E2/\00$x\07\00\80\00\00\00\FF\00\8E\07\00\E2\0F\00\12r\03\03\02\00\00\00\FF\FC\8E\07\00\C4\0F\00\12r\05\05\02\00\00\00\FF\FC\8E\07\00\E4\0F\08\12r\00\07\02\00\00\00\FF\FC\8E\07\00\E4\0F\00\11|\04\08\06\00\00\00\FF\10\8E\0F\00\E4\0F\00\11|\03\03\06\00\00\00\FF\10\8E\0F\00\E4\0F\00\11|\02\05\06\00\00\00\FF\10\8E\0F\00\E2\0F\00\88s\00\04X\00\00\00\00\0A\00\00\00\E2\03\00\11|\00\00\06\00\00\00\FF\10\8E\0F\00\C6\0F\00\88s\00\04\\ \00\00\00\0A\00\00\00\E8\03\00\88s\00\04`@\00\00\00\0A\00\00\00\E8\03\00\88s\00\04d`\00\00\00\0A\00\00\00\E8\03\00\88s\00\04h\80\00\00\00\0A\00\00\00\E8\03\00\88s\00\04l\A0\00\00\00\0A\00\00\00\E8\03\00\88s\00\04p\C0\00\00\00\0A\00\00\00\E8\03\00\88s\00\04t\E0\00\00\00\0A\00\00\00\E8\03\00\88s\00\04x\00\01\00\00\0A\00\00\00\E8\03\00\88s\00\04| \01\00\00\0A\00\00\00\E8\03\00\88s\00\04\80@\01\00\00\0A\00\00\00\E8\03\00\88s\00\04\84`\01\00\00\0A\00\00\00\E8\03\00\88s\00\04\88\80\01\00\00\0A\00\00\00\E8\03\00\88s\00\04\8C\A0\01\00\00\0A\00\00\00\E8\03\00\88s\00\04\90\C0\01\00\00\0A\00\00\00\E8\03\00\88s\00\04\94\E0\01\00\00\0A\00\00\00\E8\03\00\88s\00\03Z\00\00\00\00\0A\00\00\00\E8\03\00\88s\00\03^ \00\00\00\0A\00\00\00\E8\03\00\88s\00\03b@\00\00\00\0A\00\00\00\E8\03\00\88s\00\03f`\00\00\00\0A\00\00\00\E8\03\00\88s\00\03j\80\00\00\00\0A\00\00\00\E8\03\00\88s\00\03n\A0\00\00\00\0A\00\00\00\E8\03\00\88s\00\03r\C0\00\00\00\0A\00\00\00\E8\03\00\88s\00\03v\E0\00\00\00\0A\00\00\00\E8\03\00\88s\00\03z\00\01\00\00\0A\00\00\00\E8\03\00\88s\00\03~ \01\00\00\0A\00\00\00\E8\03\00\88s\00\03\82@\01\00\00\0A\00\00\00\E8\03\00\88s\00\03\86`\01\00\00\0A\00\00\00\E8\03\00\88s\00\03\8A\80\01\00\00\0A\00\00\00\E8\03\00\88s\00\03\8E\A0\01\00\00\0A\00\00\00\E8\03\00\88s\00\03\92\C0\01\00\00\0A\00\00\00\E8\03\00\88s\00\03\96\E0\01\00\00\0A\00\00\00\E8\03\00\88s\00\02\18\00\00\00\00\0A\00\00\00\E8\03\00\88s\00\02\1C \00\00\00\0A\00\00\00\E8\03\00\88s\00\02 @\00\00\00\0A\00\00\00\E8\03\00\88s\00\02$`\00\00\00\0A\00\00\00\E8\03\00\88s\00\02(\80\00\00\00\0A\00\00\00\E8\03\00\88s\00\02,\A0\00\00\00\0A\00\00\00\E8\03\00\88s\00\020\C0\00\00\00\0A\00\00\00\E8\03\00\88s\00\024\E0\00\00\00\0A\00\00\00\E8\03\00\88s\00\028\00\01\00\00\0A\00\00\00\E8\03\00\88s\00\02< \01\00\00\0A\00\00\00\E8\03\00\88s\00\02@@\01\00\00\0A\00\00\00\E8\03\00\88s\00\02D`\01\00\00\0A\00\00\00\E8\03\00\88s\00\02H\80\01\00\00\0A\00\00\00\E8\03\00\88s\00\02L\A0\01\00\00\0A\00\00\00\E8\03\00\88s\00\02P\C0\01\00\00\0A\00\00\00\E8\03\00\88s\00\02T\E0\01\00\00\0A\00\00\00\E8\03\00\88s\00\00\1A\00\00\00\00\0A\00\00\00\E8\03\00\88s\00\00\1E \00\00\00\0A\00\00\00\E8\03\00\88s\00\00\22@\00\00\00\0A\00\00\00\E8\03\00\88s\00\00&`\00\00\00\0A\00\00\00\E8\03\00\88s\00\00*\80\00\00\00\0A\00\00\00\E8\03\00\88s\00\00.\A0\00\00\00\0A\00\00\00\E8\03\00\88s\00\002\C0\00\00\00\0A\00\00\00\E8\03\00\88s\00\006\E0\00\00\00\0A\00\00\00\E8\03\00\88s\00\00:\00\01\00\00\0A\00\00\00\E8\03\00\88s\00\00> \01\00\00\0A\00\00\00\E8\03\00\88s\00\00B@\01\00\00\0A\00\00\00\E8\03\00\88s\00\00F`\01\00\00\0A\00\00\00\E8\03\00\88s\00\00J\80\01\00\00\0A\00\00\00\E8\03\00\88s\00\00N\A0\01\00\00\0A\00\00\00\E8\03\00\88s\00\00R\C0\01\00\00\0A\00\00\00\E8\03\00\88s\00\00V\E0\01\00\00\0A\00\00\00\E2\03\00M\09\00\00\00\00\00\00\00\00\80\03\00\EA\0F\00\19x\03\FF\1F\00\00\00\06\14\01\00\00\E2/\00\B9z\04\00\00\86\00\00\00\0A\00\00\00\E2\0F\00\12x\02\06\1F\00\00\00\FF\C0\8E\07\00\E2\0F\04Ey\00\00\C0\01\00\00\00\00\80\03\00\E2\0F\00\19x\07\06\05\00\00\00\03\12\00\00\00\E4\0F\10\19x\00\FF\05\00\00\00\03\16\01\00\00\E2\0F\00%x\02\02\10\00\00\00\FF\00\8E\07\00\E2\0F\00\10x\16\07\FC\FF\FF\FF\FF\E0\F1\07\00\C6\0F\00$x\05\07\00\02\00\00\FF\00\8E\07\00\E2\0F\04\0Cx\00\16|\00\00\00p`\F2\03\00\E4\0F\00\10x\17\00\FF\FF\FF\FF\FF\E4\7F\00\00\E4\0F\00\19x\07\07\09\00\00\00\00\02\01\00\00\E4\0F\00\0Cr\00\17\FF\00\00\00\10a\F2\03\00\E4\0F\00\12r\00\05\02\00\00\00\FF\FC\8E\07\00\E4\0F\00\12r\02\07\03\00\00\00\FF\FC\8E\07\00\C4\0F\00\10|\14\00\04\00\00\00\FF\E0\F5\0F\00\E2\0F\006|\00\00\06\00\00\00\00\00\00\08\00\E2\0F\00\1Cx\00\00\00\00\00\00p\F0\F0\03\00\E4\0F\00\10|\15\02\05\00\00\00\FF\E4\7F\09\00\E2\0F\00\B9z\04\00\00\82\00\00\00\0A\00\00\00\E4\0F\00G\99(\00\00\00\00\00\00\00\80\03\00\F0\0F\00\84y\04\00\00\00\00\00\00\0C\00\00\00\A2\02\00$r\02\FF\FF\00\00\00\14\00\8E\07\00\E2\0F\00\10x\16\16\04\00\00\00\FF\E0\F3\07\00\E2\0F\00$r\03\FF\FF\00\00\00\15\00\8E\07\00\E2\0F\00\10x\14\14\00\08\00\00\FF\E0\F5\07\00\E4\0F\00\1Cx\00\00\00\00\00\00p\E1\F0\03\00\E2\0F\00$r\17\FF\FF\00\00\00\17\06\8E\00\00\E4\0F\006x\00\00\00\08\00\00\00\00\00\00\00\E4/\00$r\15\FF\FF\00\00\00\15\06\0E\01\00\E2\0F\00\86y\00\02\04\00\00\00\04\1D\10\0C\00\EEC\00Ay\00\00\00\00\00\00\00\00\80\03\00\EA\0F\00\10x\02\16|\00\00\00\FF\E1\F7\07\00\E2/\00Ey\00\00\A0\01\00\00\00\00\80\03\00\E2\0F\00\0Cx\00\16|\00\00\00p`\F4\03\00\E4\0F\00\0Cx\00\02\0C\00\00\00p0\F2\03\00\E2\0F\00$r\02\FF\FF\00\00\00\17\0E\8E\01\00\E2\0F\00\0Cr\00\17\FF\00\00\00 a\F4\03\00\C8\0F\00\0Cr\00\02\FF\00\00\00\105r\01\00\DA\0F\00G\19L\00\00\00\00\00\00\00\80\03\00\EA\0F\00\1Cx\00\00\00\00\00\00p\E1\F0\03\00\DA\0F\00\84y\04\00\00\00\00\00\00\0C\00\00\00b.\00$r\02\FF\FF\00\00\00\14\00\8E\07\00\E2\0F\00\10x\16\16\10\00\00\00\FF\E0\F3\07\00\E2\0F\00$r\03\FF\FF\00\00\00\15\00\8E\07\00\E2\0F\00\84y\08\00\00\00\08\00\00\0C\00\00\00\A4\0E\00\10x\14\02\00 \00\00\FF\E0\F5\07\00\E2\0F\00$r\17\FF\FF\00\00\00\17\06\8E\00\00\E2\0F\00\84y\0C\00\00\00\10\00\00\0C\00\00\00\E2\0E\00\0Cx\00\16p\00\00\00p`\F2\03\00\E4\0F\00$r\15\FF\FF\00\00\00\03\06\0E\01\00\E2\0F\00\84y\10\00\00\00\18\00\00\0C\00\00\00b\09\00\0Cr\00\17\FF\00\00\00\10a\F2\03\00\E2\0F\006x\00\00\00 \00\00\00\00\00\00\00\C4\0F\01\86y\00\02\04\00\00\00\04\1D\10\0C\00\E8#\00\86y\00\02\08\00\08\00\04\1D\10\0C\00\E8C\00\86y\00\02\0C\00\10\00\04\1D\10\0C\00\E8\83\00\86y\00\02\10\00\18\00\04\1D\10\0C\00\E2\03\02G\99\B8\00\FC\FF\FF\FF\FF\FF\83\03\00\EA\0F\00Ay\00\00\00\00\00\00\00\00\80\03\00\EA\0F\00\10x\02\16|\00\00\00\FF\E1\F7\07\00\E2/\00Ey\00\000\01\00\00\00\00\80\03\00\E2\0F\00\0Cx\00\16|\00\00\00p`\F4\03\00\E4\0F\00\0Cx\00\02\04\00\00\00p0\F2\03\00\E2\0F\00$r\02\FF\FF\00\00\00\17\0E\8E\01\00\E2\0F\00\0Cr\00\17\FF\00\00\00 a\F4\03\00\C8\0F\00\0Cr\00\02\FF\00\00\00\105r\01\00\DA\0F\00G\190\00\00\00\00\00\00\00\80\03\00\EA\0F\00\84y\04\00\00\00\00\00\00\0C\00\00\00b\0E\00$r\02\FF\FF\00\00\00\14\00\8E\07\00\E2\0F\00\10x\16\16\08\00\00\00\FF\E0\F3\07\00\E2\0F\00$r\03\FF\FF\00\00\00\15\00\8E\07\00\E2\0F\00\84y\08\00\00\00\08\00\00\0C\00\00\00\E2\04\00\10x\14\14\00\10\00\00\FF\E0\F5\07\00\E4\0F\00\1Cx\00\00\00\00\00\00p\E1\F0\03\00\E2\0F\00$r\17\FF\FF\00\00\00\17\06\8E\00\00\E4\0F\00$r\15\FF\FF\00\00\00\15\06\0E\01\00\C4\0F\006x\00\00\00\10\00\00\00\00\00\00\00\E2O\00\86y\00\02\04\00\00\00\04\1D\10\0C\00\E8#\00\86y\00\02\08\00\08\00\04\1D\10\0C\00\E6\83\00Ay\00\00\00\00\00\00\00\00\80\03\00\EA\0F\00\0Cx\00\16|\00\00\00p\10\F2\03\00\C8\0F\00\0Cr\00\17\FF\00\00\00\10\15p\00\00\DA\0F\00M\89\00\00\00\00\00\00\00\00\80\03\00\EA\0F\00\84y\04\00\00\00\00\00\00\0C\00\00\00\22.\00$r\02\FF\FF\00\00\00\14\00\8E\07\00\E4\0F\00$r\03\FF\FF\00\00\00\15\00\8E\07\00\CA\0F\00\86y\00\02\04\00\00\00\04\1D\10\0C\00\E2\1F\00My\00\00\00\00\00\00\00\00\80\03\00\EA\0F\00Gy\FC\00\FC\FF\FF\FF\FF\FF\83\03\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\18y\00\00\00\00\00\00\00\00\00\00\00\C0\0F\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00@\00\00\00\00\00\00\00\F5\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\0B\00\00\00\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00`\01\00\00\00\00\00\00L\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\13\00\00\00\02\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\B0\02\00\00\00\00\00\00\A8\00\00\00\00\00\00\00\02\00\00\00\05\00\00\00\08\00\00\00\00\00\00\00\18\00\00\00\00\00\00\00\AB\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00X\03\00\00\00\00\00\00h\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\007\00\00\00\00\00\00p\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\C0\03\00\00\00\00\00\000\00\00\00\00\00\00\00\03\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00R\00\00\00\00\00\00p\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\F0\03\00\00\00\00\00\00\FC\00\00\00\00\00\00\00\03\00\00\00\09\00\00\00\04\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\94\00\00\00\04\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\F0\04\00\00\00\00\00\00\00\00\00\00\00\00\00\00\03\00\00\00\09\00\00\00\08\00\00\00\00\00\00\00\18\00\00\00\00\00\00\00\C9\00\00\00\04\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\F0\04\00\00\00\00\00\00\18\00\00\00\00\00\00\00\03\00\00\00\04\00\00\00\08\00\00\00\00\00\00\00\18\00\00\00\00\00\00\00@\00\00\00\01\00\00\00\06\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\80\05\00\00\00\00\00\00\80\18\00\00\00\00\00\00\03\00\00\00\05\00\00\00\80\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00g\00\00\00\08\00\00\00\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\1E\00\00\00\00\00\00!\04\00\00\00\00\00\00\00\00\00\00\09\00\00\00\10\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\DB\00\00\00\01\00\00\00\02\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\1E\00\00\00\00\00\00X\02\00\00\00\00\00\00\00\00\00\00\09\00\00\00\04\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\06\00\00\00\04\00\00\00X#\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\18\01\00\00\00\00\00\00\18\01\00\00\00\00\00\00\08\00\00\00\00\00\00\00\01\00\00\00\04\00\00\00X#\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\18\01\00\00\00\00\00\00\18\01\00\00\00\00\00\00\08\00\00\00\00\00\00\00\01\00\00\00\05\00\00\00\80\05\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\80\18\00\00\00\00\00\00\80\18\00\00\00\00\00\00\08\00\00\00\00\00\00\00\01\00\00\00\06\00\00\00\00\1E\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00!\04\00\00\00\00\00\00\08\00\00\00\00\00\00\00\01\00\00\00\04\00\00\00\00\1E\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00X\02\00\00\00\00\00\00X\02\00\00\00\00\00\00\08\00\00\00\00\00\00\00\01\00\01\01H\00\00\00\B0\0E\00\00\00\00\00\00\AD\0E\00\00@\00\00\00\00\00\08\00Z\00\00\00\00\00\00\00\00\00\00\00\11 \10\00\00\00\00\00\00\00\00\00\00\00\00\00\F7=\00\00\00\00\00\00\00\00\00\00\00\00\00\00\F2\22\0A\0A\0A\0A.version 8.0\0A.target sm_90a\0A.address_size 64\0A\01\00\F8\19.visible .entry main_kernel(\0A.param .u64\19\00\11_\17\00?_0,!\00\0C\1F1!\00\0D\1F2!\00\0D\1F3!\00\0D\1F4!\00\0D\1F5!\00\0D\1F6!\00\0D\1F7!\00\0D\F3\088\0A)\0A{\0A.reg .pred %p<8>;\12\00\95b32 %r<47\12\00\10f\12\00ff<1669&\00\F0\0364 %rd<83>;\0A\0A\09.shaJ\00\FF\0B.align 16 .b8 dynamicShmem&\00\00\118%\00\EF__mbarrier[16]M\00\07$acI\00\22ld\E0\00\22.u\85\00_21, [\E7\00\00\1F]+\00\00\1F0+\00\02\1F7+\00\00/17+\00\02\911];\0Amov.u'\01\F0\051, %tid.x;\0Asetp.ne.s\19\002p2,\1E\00\130.\00\03T\00'8,\F0\00\02\1B\00\02p\01d2, 1;\0A\1A\00S.init\07\01\01k\01\11[=\00\10]U\00\832;\0Aadd.sR\00$9,Y\00\1F8@\00\0D\149@\00\F1\05\0A\09prefetch.tensormap#\00!20\81\01\0F \00\07\111 \00\07\D2\00;78,\E7\01\06\D4\00\116\FF\00\F0\08@%p2 bra $L__BB0_2;\0AcvtA\01\03E\00\133\C3\00\08<\00\814, 32768o\00\04\D6\00\00\07\00\C8ve.expect_tx\E2\00 _,\A2\00\113\E3\00\1046\00\0Ad\00\225,\AE\00\01\1A\00\C3p.async.bulk\E2\003.2dN\00\F4\02::cluster.global.\7F\00\F0\05::complete_tx::bytest\0035],=\010, {\F6\00c%r6} ]\90\00\00q\00\01\B0\01\01\D7\00\01\AE\01\175\DC\00\0F\8C\00<\149\8C\00\1F1\8C\00\0F#13\8D\00X40960}\01o14, 64\9F\00@*13\A0\00/14\A1\00\09222,\1E\02\0F\06\02\1D&22\07\02\07\F0\00\04}\01O1638\DE\00B\1A9\0A\02\0A\DE\00,22\DF\00\05\80\01O9152\90\00@.23\0E\02/14\90\00\07\137\90\00O5734 \01A*27\90\00/14\91\00\00\05&\04\10:&\04Da.to_\00\02\C9\031d1,\06\00\127F\043s64\BF\00\12ds\05\02\C8\02\02\B3\060154\83\05#f0\01\00\09\8C\05%79\9E\05\03\05\07X7, -1\08\03o33, 10<\00\00\104F\02\09<\00\154M\00\05{\00#2,\83\00\0B\18\00\1F3\18\00\04\1F4\18\00\04\1F5\18\00\04\1F6\18\00\04\1F7\18\00\04\1F8\18\00\04\1F9\18\00\03/50\18\00\04\1F1\18\00\04\0F\F0\00\04\1F5\F0\00\04\1F5\F0\00\04\1F5\F0\00\04\1F5\F0\00\04\1F5\F0\00\04\1F5\F0\00\04\1F5\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F6\F0\00\04\1F7\F0\00\04\1F7\F0\00\04\1F7\F0\00\04\1F7\F0\00\04\1F7\F0\00\04\1F7\F0\00\04\1F7\F0\00\04\1F7\F0\00\04/78\18\00\04\0F\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F8\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\04\1F9\F0\00\03?600\18\00\04\1F1\18\00\04\1F2\18\00\04\1F3\18\00\04\1F4\18\00\04\1F5\18\00\04\1F6\18\00\04\1F7\18\00\04\1F8\18\00\04\0F\F0\00\04\1F1\F0\00\04\1F1\F0\00\04\1F1\F0\00\04/13\18\00\04\0F\F0\00\04\1F1\F0\00\04\1F1\F0\00\04/17\18\00\04\0F\F0\00\04/19\18\00\03/20\18\00\04\0F\F0\00\04/22\18\00\04\0F\F0\00\04\1F2\F0\00\04\1F2\F0\00\04\1F2\F0\00\04/27\18\00\04\0F\F0\00\04\1F2\F0\00\04\1F3\F0\00\04\1F3\F0\00\04\1F3\F0\00\04/33\18\00\04\0F\F0\00\04\1F3\F0\00\04\1F3\F0\00\04\1F3\F0\00\04\1F3\F0\00\04\1F3\F0\00\04\1F4\F0\00\04/41\18\00\04\0F\F0\00\04\0F`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\1F6`\09\04\186`\09\04\A5\0C(3:\04\0C\00\C9\00\93p7;\0Ashl.b)\0C\00e\00\02l\0C\193\B8\11\01.\02\02\B8\11\01'\00\0D\14\11\121\19\00\01\FB\0E\09\AD\13\F6\00P1; \0ALAB_WAIT: \F4\11\80try_wait\C7\128ity\17\0F!P1b\0D!31\18\0F\01\94\00\8033; \0A@P1\9B\11\C4.uni DONE; \0A\0F\00\04b\000; \0A\1A\00x: \0A}\0A\0A\09\DB\00\159\DB\00)14\DC\00\01\C2\02\02\1B\12\01(\00T;\0Abfe;\0D\01\C7\02\02$\00\124:\00$orS\00\01\0B\05\02#\00\FA\03461168629337240371\F8\12%53G\00\03L\11\08d\00$4,$\00\0Dd\00\01W\05\02#\00\07d\00\8038849280n\01\C1wgmma.fence.\FE\0E\01\87\14\22ed\1A\13\09\8C\01\16p\F7\13\002\0E2p, \F3\13\02C\00Bmma_@\0F\08G\00\B0.m64n128k16\\\02 .f\08\00B16 {]\02\1A,=\0E*3,\1D\0E*5,\FD\0D*7,\DD\0D)9,\BD\0D\00 \01\08\9D\0D\017\01\07}\0D\04\0D\04\04]\0D\04\ED\03\04=\0D\04\CD\03\04\1D\0D\04\AD\03\04\FD\0C\04\8D\03\04\DD\0C\04m\03\04\BD\0C\04M\03\04\9D\0C\04\8D\0C\04}\0C\04m\0C\04]\0C\04M\0C\04=\0C\04-\0C\04\1D\0C\04\0D\0C\04\FD\0B\00~\02\08\DD\0B\04\CD\0B\04\BD\0B\04\AD\0B\04\9D\0B\04\8D\0B\04}\0B\04m\0B\04]\0B\04M\0B\04=\0B\04-\0B\04\1D\0B\04\0D\0B\04\FD\0A\04\ED\0A\04\DD\0A\04\CD\0A\04\BD\0A\129P\00\04\9D\0A\04\8D\0A\04}\0A\04m\0AW604},\02\03\00\A5\02\02M\02\00P\02\01S\02$}\0A\AD\11\03\C8\02\1F1,\03\08\0A\90\03/32\F3\02\06.40a\04\0F\D5\02\FF\FF@\03\AB\02\00\87\02\0F\D5\02\0D\1F3\D5\02\08\1A6\D5\02\1F4\D5\02\06?536\D5\02\FF\FFR$3,\87\02\0F\D5\02\0D\1F5\D5\02\08\1A8\D5\02\1F6\D5\02\06?664\D5\02\FF\FFR$5,\87\02\0F\D5\02\0D\1F7\D5\02\05?422\AA\02O\04v\13\04f\13\04V\13\04F\13\046\13\04&\13\04\16\13\04\06\13\04\F6\12\04\E6\12\04\D6\12\04\C6\12\04\B6\12\04\A6\12\04\96\12\04\86\12\04v\12\04f\12\04V\12\04F\12\046\12\04&\12\04\16\12\04\06\12\04\F6\11\04\E6\11\04\D6\11\04\C6\11\04\B6\11\04\A6\11\04\96\11\04\86\11\04v\11\04f\11\04V\11\136F\11\046\11\04&\11\04\16\11\04\06\11\04\F6\10\04\E6\10\04\D6\10\04\C6\10\04\B6\10\04\A6\10\04\96\10\04\86\10\04v\10\04f\10\04V\10\04F\10\046\10\04&\10\04\16\10\04\06\10\04\F6\0F\04\E6\0F\04\D6\0F\04\C6\0F\04\B6\0F\04\A6\0F\04\96\0F3668\AA\02\1F7)\0B\16\1F9\AA\02\08\0F)\08O\0F\AA\02\FF\F0\1F9\FE\0A\15\01\0A\01\0F\AA\02\05\0F\A8\0DO\0F\AA\02\FF\EF/41\D3\0A\15\01\FA\00\0F\AA\02\04/30\AA\02\FF\FFR\1F3\A8\0A\0A\04\C4\15\CFcommit_group\CB\15\00\02%\00/wa#\00\02\00\E0#*\0A\09\1E$=%p4C$\00\8E\00\115\E6(\191\E6(\1D3\D2\17\114-\00\01\E9%\0Fz\00\0B\01A\03\01\CF\16\00{\18\11r\FD\01\01?\00\112F\18\01_\18\02\1A\00\146\1A\00d1;\0Aand\17\00#7,\1D\00\106.\00\14rH\00\1F8.\00\03#9,\1D\00\A221474836321\17\01\1F\00\01S\02\04\84\00\139\D5\00#64T\00\22d5\19\00\B57;\0Amul.wide\1A\00#6,?\008128\82\17\01\1D\02\04 \003d55\D2\00\03\1D\00$8,$\00\192l\01\00C\02\06\03,\07\8C\04\01T\02\02 \00\00\07\00E8;\0As\FB* v2<\04! ['\00!],=\04\07\E5\0E\1F},\00\065+32/\00\05\F4\0E\0F/\00\08%64/\00\05\03\0F\0F/\00\08$96/\00\06\12\0F\0F/\00\0851280\00\05\22\0F\0F0\00\09\06\ED\00\062\0F\0F0\00\09\159\EE\00\06B\0F\0F0\00\08%22\EF\00\06R\0F\0F0\00\09\155\F0\00\06b\0F\0F0\00\09\158\F0\00\06r\0F\0F\AD\01\0A\05\F0\00\06\82\0F\0F0\00\09\155\F0\00\06\92\0F\0F0\00\09\158\F0\00\06\A2\0F\0F0\00\08%41\F0\00\06\B2\0F\0F0\00\09\154\F0\00\06\C2\0F\0F0\00\09\148\F0\00\07\D2\0F\18}\B5\03\141|\03\1E8\99\03\01\13\02\01#\00\0B\99\03\01\8E\05\04 \00\0C\99\03\01\A3\05\02$\00\0A\8A\1B\01\B4\05\05\80\03/63\B7\00\05\05C\01\06U\12\0F\E3\00\06\184\80\03\05d\12\0F/\00\08\06^\00\06s\12\0F/\00\08\07\80\03\05\82\12\0F/\00\08\08\80\03\05\92\12\0F0\00\09\07\80\03\05\A2\12\0F0\00\09\07\80\03\05\B2\12\0F0\00\08\07\80\03\06\C2\12\0F0\00\09\07\80\03\05\D2\12\0F0\00\09\07\80\03\05\E2\12\0F\AD\01\0A\05\F0\00\06\F2\12\0F0\00\09\07\80\03\05\02\13\0F0\00\09\06\80\03\06\12\13\0F0\00\08\08\80\03\05\22\13\0F0\00\09\07\80\03\052\13\0F0\00\09\07\80\03\06B\13&;\0AL-\01y\09\01\81\03\00H/\05\82\03\03\AA,\01e\05\01$\00\0C\82\03&6, \00\0D\82\03$7,$\00\0B\82\03\198\82\03\1F7\B9\00\05\04\E5\00\07?\0B\0F\E5\00\06\168\82\03\07N\0B\0F/\00\08\05\82\03\07]\0B\0F/\00\08\05\82\03\07l\0B\0F/\00\08\06\82\03\07|\0B\0F0\00\09\05\82\03\07\8C\0B\0F0\00\09\05\82\03\07\9C\0B\0F0\00\08\06\82\03\07\AC\0B\0F0\00\09\05\82\03\07\BC\0B\0F0\00\09\05\82\03\07\CC\0B\0F\AD\01\0A\05\F0\00\06\DC\0B\0F0\00\09\05\82\03\07\EC\0B\0F0\00\09\05\82\03\07\FC\0B\0F0\00\08\06\82\03\07\0C\0C\0F0\00\09\05\82\03\07\1C\0C\0F0\00\09\06\82\03\06,\0C\0A\82\03\143\82\03/72\82\03\00#9,$\00\0B\82\03\01\1A\18\04 \00\0C\82\03\01m\05\02$\00\0A\82\03\01@\18\05\82\03/71\B9\00\04\157u\01\06\B1\0E\1F},\00\06\07\82\03\06\C0\0E\0F/\00\08\07\82\03\05\CF\0E\0F/\00\08\07\82\03\05\DE\0E\0F/\00\08\08\82\03\05\EE\0E\0F0\00\09\07\82\03\05\FE\0E\0F0\00\09\06\82\03\06\0E\0F\0F0\00\08\08\82\03\05\1E\0F\0F0\00\09\07\82\03\05.\0F\0F0\00\09\07\82\03\05>\0F\0F\AD\01\0A\05\F0\00\06N\0F\0F0\00\09\06\82\03\06^\0F\0F0\00\09\07\82\03\05n\0F\0F0\00\08\08\82\03\05~\0F\0F0\00\09\07\82\03\05\8E\0F\0F0\00\09\07\82\03\06\9E\0F\03\E9\11\11g\F9\0ED %p5s\0E2409&\0F\195&\0F\137\A1\0E\02\E4\0D\01\1D\0C\01\C93\195_\03\01O\1B\02\1F\00*-4\92\03\154\1A\00\189\D5\0E\154y\00.31\02\0B\01\05\09\01$\00)16\FD\03(6,U\00\0B\85\00&1,\7F4\1A7\8D \01\00\1C\06\00\04\166\22( 6:Y:\04B\01\114B\01\005\01\135\07\063538\08\00\02\A5\01A540}\AC'Ad80]~\01\03\FE4%v4~\01#81z\01\0FE\00\09\0FA\01\01\00\07\00\1F4\D5\00\01\00\07\00O2048\D5\00\01$80\1C\00\01\DB\01\11l\FF\0FE %p6S\001124\DB\01\196\DB\01\07\05\01\C07:\0Aret;\0A\0A}\0A\00\00\00\00">] + llvm.func @mgpuTensorMapEncodeTiledMemref(i64, !llvm.ptr, i64, i64, i64, i64, i64, !llvm.ptr) -> !llvm.ptr + llvm.func @mgpuStreamCreate() -> !llvm.ptr + llvm.func @mgpuMemAlloc(i64, !llvm.ptr, i8) -> !llvm.ptr + llvm.func @mgpuMemcpy(!llvm.ptr, !llvm.ptr, i64, !llvm.ptr) +} + diff --git a/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir b/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir index bca3cb1f9a1e..025282ec0d68 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/cga_cluster.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3" \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir b/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir index c8dc45ab861d..35ca0ee8677c 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir @@ -1,5 +1,6 @@ // RUN: mlir-opt %s \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3" \ +// RUN: -convert-linalg-to-loops \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir b/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir index bc3437b6545d..5a10bbba26d8 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir @@ -1,5 +1,6 @@ // RUN: mlir-opt %s \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3" \ +// RUN: -convert-linalg-to-loops \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_90a cubin-features=+ptx80 opt-level=3" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir index 65f301968669..9c5aacf96b0d 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_128x64_swizzle128b.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_90 cubin-features=+ptx80 opt-level=3" \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_90 cubin-features=+ptx80 opt-level=3" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir index fdbb188c28a9..536e71d260f5 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x64_swizzle128b.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_90 cubin-features=+ptx80 opt-level=3" \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_90 cubin-features=+ptx80 opt-level=3" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir index ed58504cfdb1..aee265e3faf1 100644 --- a/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir +++ b/mlir/test/Integration/GPU/CUDA/sm90/tma_load_64x8_8x128_noswizzle.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: -gpu-lower-to-nvvm="cubin-chip=sm_90 cubin-features=+ptx80 opt-level=3" \ +// RUN: -gpu-lower-to-nvvm-pipeline="cubin-chip=sm_90 cubin-features=+ptx80 opt-level=3" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ diff --git a/mlir/test/Integration/GPU/CUDA/two-modules.mlir b/mlir/test/Integration/GPU/CUDA/two-modules.mlir index f68359d78c04..db4b365dd85d 100644 --- a/mlir/test/Integration/GPU/CUDA/two-modules.mlir +++ b/mlir/test/Integration/GPU/CUDA/two-modules.mlir @@ -1,5 +1,5 @@ // RUN: mlir-opt %s \ -// RUN: | mlir-opt -gpu-lower-to-nvvm="cubin-format=%gpu_compilation_format" \ +// RUN: | mlir-opt -gpu-lower-to-nvvm-pipeline="cubin-format=%gpu_compilation_format" \ // RUN: | mlir-cpu-runner \ // RUN: --shared-libs=%mlir_cuda_runtime \ // RUN: --shared-libs=%mlir_runner_utils \ -- GitLab From 20a05677f9394d4bc9467fe7bc93a4ebd3aeda61 Mon Sep 17 00:00:00 2001 From: Mariya Podchishchaeva Date: Fri, 5 Jan 2024 15:14:51 +0300 Subject: [PATCH 099/728] [clang] Accept recursive non-dependent calls to functions with deduced return type (#75456) Treat such calls as dependent since it is much easier to implement. Fixes https://github.com/llvm/llvm-project/issues/71015 --- clang/docs/ReleaseNotes.rst | 3 ++ clang/lib/AST/ComputeDependence.cpp | 2 + clang/lib/Sema/SemaOverload.cpp | 18 ++++++++ .../SemaCXX/deduced-return-type-cxx14.cpp | 46 +++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 903a50a4d4d3..cd7d50455187 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -692,6 +692,9 @@ Bug Fixes in This Version Fixes (`#64347 `_) - Fix crash when using C++ only tokens like ``::`` in C compiler clang. Fixes (`#73559 `_) +- Clang now accepts recursive non-dependent calls to functions with deduced + return type. + Fixes (`#71015 `_) Bug Fixes to Compiler Builtins diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp index 097753fd3267..584b58473294 100644 --- a/clang/lib/AST/ComputeDependence.cpp +++ b/clang/lib/AST/ComputeDependence.cpp @@ -603,6 +603,8 @@ ExprDependence clang::computeDependence(PredefinedExpr *E) { ExprDependence clang::computeDependence(CallExpr *E, llvm::ArrayRef PreArgs) { auto D = E->getCallee()->getDependence(); + if (E->getType()->isDependentType()) + D |= ExprDependence::Type; for (auto *A : llvm::ArrayRef(E->getArgs(), E->getNumArgs())) { if (A) D |= A->getDependence(); diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp index 5026e1d603e5..9fb767101e1e 100644 --- a/clang/lib/Sema/SemaOverload.cpp +++ b/clang/lib/Sema/SemaOverload.cpp @@ -13994,6 +13994,24 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, OverloadCandidateSet::iterator Best; OverloadingResult OverloadResult = CandidateSet.BestViableFunction(*this, Fn->getBeginLoc(), Best); + FunctionDecl *FDecl = Best->Function; + + // Model the case with a call to a templated function whose definition + // encloses the call and whose return type contains a placeholder type as if + // the UnresolvedLookupExpr was type-dependent. + if (OverloadResult == OR_Success && FDecl && + FDecl->isTemplateInstantiation() && + FDecl->getReturnType()->isUndeducedType()) { + if (auto TP = FDecl->getTemplateInstantiationPattern(false)) { + if (TP->willHaveBody()) { + CallExpr *CE = + CallExpr::Create(Context, Fn, Args, Context.DependentTy, VK_PRValue, + RParenLoc, CurFPFeatureOverrides()); + result = CE; + return result; + } + } + } return FinishOverloadedCallExpr(*this, S, Fn, ULE, LParenLoc, Args, RParenLoc, ExecConfig, &CandidateSet, &Best, diff --git a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp index 6344d1df3fba..eac9c587869f 100644 --- a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp +++ b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp @@ -640,3 +640,49 @@ namespace PR46637 { template struct Y { T x; }; Y auto> y; // expected-error {{'auto' not allowed in template argument}} } + +namespace GH71015 { + +// Check that there is no error in case a templated function is recursive and +// has a placeholder return type. +struct Node { + int value; + Node* left; + Node* right; +}; + +bool parse(const char*); +Node* parsePrimaryExpr(); + +auto parseMulExpr(auto node) { // cxx14-error {{'auto' not allowed in function prototype}} \ + // cxx14-note {{not viable}} + if (node == nullptr) node = parsePrimaryExpr(); + if (!parse("*")) return node; + return parseMulExpr(new Node{.left = node, .right = parsePrimaryExpr()}); +} + +template +auto parseMulExpr2(T node) { + if (node == nullptr) node = parsePrimaryExpr(); + if (!parse("*")) return node; + return parseMulExpr2(new Node{.left = node, .right = parsePrimaryExpr()}); +} + +template +auto parseMulExpr3(T node) { // expected-note {{declared here}} + if (node == nullptr) node = parsePrimaryExpr(); + return parseMulExpr3(new Node{.left = node, .right = parsePrimaryExpr()}); // expected-error {{cannot be used before it is defined}} +} + +void foo() { + parseMulExpr(new Node{}); // cxx14-error {{no matching function}} + parseMulExpr2(new Node{}); + parseMulExpr3(new Node{}); // expected-note {{in instantiation}} +} + +auto f(auto x) { // cxx14-error {{'auto' not allowed in function prototype}} + if (x == 0) return 0; + return f(1) + 1; +} + +} -- GitLab From 7648371c25cf21b84028935c298e2dc088ae5a52 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 5 Jan 2024 12:27:24 +0000 Subject: [PATCH 100/728] =?UTF-8?q?Revert=204d7c5ad58467502fcbc433591edff4?= =?UTF-8?q?0d8a4d697d=20"[NewPM]=20Update=20CodeGenPreparePass=20reference?= =?UTF-8?q?=20in=20CodeGenPassBuilder=20(#77054)"=20Revert=20e0c554ad87d18?= =?UTF-8?q?dcbfcb9b6485d0da800ae1338d1=20"Port=20CodeGenPrepare=20to=20new?= =?UTF-8?q?=20pass=20manager=20(and=20BasicBlockSectionsProfil=E2=80=A6=20?= =?UTF-8?q?(#75380)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert #75380 and #77054 as they were breaking EXPENSIVE_CHECKS buildbots: https://lab.llvm.org/buildbot/#/builders/104 --- .../CodeGen/BasicBlockSectionsProfileReader.h | 83 +++--------- .../include/llvm/CodeGen/CodeGenPassBuilder.h | 3 +- llvm/include/llvm/CodeGen/CodeGenPrepare.h | 35 ----- .../llvm/CodeGen/MachinePassRegistry.def | 2 +- llvm/include/llvm/CodeGen/Passes.h | 4 +- llvm/include/llvm/InitializePasses.h | 4 +- llvm/include/llvm/LinkAllPasses.h | 2 +- llvm/lib/CodeGen/BasicBlockPathCloning.cpp | 11 +- llvm/lib/CodeGen/BasicBlockSections.cpp | 8 +- .../BasicBlockSectionsProfileReader.cpp | 52 ++------ llvm/lib/CodeGen/CodeGen.cpp | 2 +- llvm/lib/CodeGen/CodeGenPrepare.cpp | 124 ++++++------------ llvm/lib/CodeGen/TargetPassConfig.cpp | 4 +- llvm/lib/Passes/PassBuilder.cpp | 2 - llvm/lib/Passes/PassRegistry.def | 2 - .../AArch64/aarch64-codegen-prepare-atp.ll | 2 +- llvm/test/CodeGen/AArch64/and-sink.ll | 4 +- .../CodeGen/AArch64/arm64-bitfield-extract.ll | 2 +- .../AArch64/arm64-codegen-prepare-extload.ll | 6 +- .../test/CodeGen/AArch64/arm64_32-gep-sink.ll | 2 +- .../CodeGen/AArch64/cgp-trivial-phi-node.ll | 2 +- llvm/test/CodeGen/AArch64/convertphitype.ll | 2 +- .../AArch64/scalable-vector-promotion.ll | 2 +- llvm/test/CodeGen/AArch64/sve-vscale.ll | 2 +- .../CodeGen/AArch64/sve2-vscale-sinking.ll | 2 +- .../AMDGPU/cgp-addressing-modes-flat.ll | 8 +- .../AMDGPU/cgp-addressing-modes-gfx1030.ll | 2 +- .../AMDGPU/cgp-addressing-modes-gfx908.ll | 2 +- .../CodeGen/AMDGPU/cgp-addressing-modes.ll | 8 +- llvm/test/CodeGen/ARM/vector-promotion.ll | 4 +- .../Generic/addr-sink-call-multi-arg.ll | 2 +- llvm/test/CodeGen/Generic/addr-use-count.ll | 2 +- .../test/CodeGen/X86/callbr-codegenprepare.ll | 2 +- .../X86/codegen-prepare-addrmode-sext.ll | 2 +- .../CodeGen/X86/codegen-prepare-extload.ll | 6 +- llvm/test/CodeGen/X86/convertphitype.ll | 2 +- .../CodeGen/X86/indirect-br-gep-unmerge.ll | 2 +- llvm/test/CodeGen/X86/pr58538.ll | 4 +- llvm/test/CodeGen/X86/tailcall-cgp-dup.ll | 2 +- llvm/test/CodeGen/X86/tailcall-extract.ll | 2 +- llvm/test/DebugInfo/ARM/salvage-debug-info.ll | 2 +- llvm/test/DebugInfo/X86/zextload.ll | 2 +- llvm/test/Other/codegenprepare-and-debug.ll | 2 +- .../AArch64/combine-address-mode.ll | 2 +- .../CodeGenPrepare/AArch64/free-zext.ll | 2 +- .../gather-scatter-opt-inseltpoison.ll | 2 +- .../AArch64/gather-scatter-opt.ll | 2 +- .../AArch64/overflow-intrinsics.ll | 6 +- .../AArch64/sink-gather-scatter-addressing.ll | 2 +- .../AArch64/trunc-weird-user.ll | 2 +- .../CodeGenPrepare/AArch64/zext-to-shuffle.ll | 2 +- .../CodeGenPrepare/AMDGPU/addressing-modes.ll | 2 +- .../AMDGPU/no-sink-addrspacecast.ll | 2 +- .../AMDGPU/sink-addrspacecast.ll | 2 +- .../CodeGenPrepare/ARM/branch-on-zero.ll | 2 +- .../Transforms/CodeGenPrepare/ARM/dead-gep.ll | 2 +- .../CodeGenPrepare/ARM/memory-intrinsics.ll | 2 +- .../CodeGenPrepare/ARM/overflow-intrinsics.ll | 2 +- .../CodeGenPrepare/ARM/sink-addrmode.ll | 2 +- .../Transforms/CodeGenPrepare/ARM/splitgep.ll | 2 +- .../CodeGenPrepare/ARM/tailcall-dup.ll | 2 +- .../bypass-slow-div-constant-numerator.ll | 2 +- .../NVPTX/bypass-slow-div-not-exact.ll | 2 +- .../NVPTX/bypass-slow-div-special-cases.ll | 2 +- .../CodeGenPrepare/NVPTX/bypass-slow-div.ll | 2 +- .../NVPTX/dont-introduce-addrspacecast.ll | 2 +- .../NVPTX/dont-sink-nop-addrspacecast.ll | 2 +- .../PowerPC/split-store-alignment.ll | 4 +- .../CodeGenPrepare/RISCV/and-mask-sink.ll | 8 +- .../CodeGenPrepare/RISCV/cttz-ctlz.ll | 2 +- .../SPARC/overflow-intrinsics.ll | 6 +- .../CodeGenPrepare/X86/catchpad-phi-cast.ll | 4 +- .../X86/cgp_shuffle_crash-inseltpoison.ll | 2 +- .../CodeGenPrepare/X86/cgp_shuffle_crash.ll | 2 +- .../CodeGenPrepare/X86/computedgoto.ll | 2 +- .../CodeGenPrepare/X86/cttz-ctlz.ll | 10 +- .../X86/delete-assume-dead-code.ll | 2 +- .../CodeGenPrepare/X86/extend-sink-hoist.ll | 2 +- .../CodeGenPrepare/X86/freeze-brcond.ll | 2 +- .../X86/gather-scatter-opt-inseltpoison.ll | 4 +- .../CodeGenPrepare/X86/gather-scatter-opt.ll | 2 +- .../CodeGenPrepare/X86/gep-unmerging.ll | 2 +- .../CodeGenPrepare/X86/invariant.group.ll | 2 +- .../X86/masked-gather-struct-gep.ll | 2 +- .../CodeGenPrepare/X86/nonintegral.ll | 4 +- .../CodeGenPrepare/X86/optimizeSelect-DT.ll | 2 +- .../CodeGenPrepare/X86/overflow-intrinsics.ll | 6 +- .../Transforms/CodeGenPrepare/X86/pr27536.ll | 2 +- .../Transforms/CodeGenPrepare/X86/pr35658.ll | 2 +- .../Transforms/CodeGenPrepare/X86/pr72046.ll | 2 +- .../recursively-delete-dead-instructions.ll | 2 +- .../CodeGenPrepare/X86/remove-assume-block.ll | 2 +- .../Transforms/CodeGenPrepare/X86/select.ll | 6 +- .../CodeGenPrepare/X86/sink-addrmode-base.ll | 4 +- .../X86/sink-addrmode-inseltpoison.ll | 2 +- .../X86/sink-addrmode-select.ll | 2 +- .../X86/sink-addrmode-two-phi.ll | 2 +- .../CodeGenPrepare/X86/sink-addrmode.ll | 2 +- .../CodeGenPrepare/X86/sink-addrspacecast.ll | 2 +- .../CodeGenPrepare/X86/split-indirect-loop.ll | 2 +- .../X86/split-store-alignment.ll | 2 +- .../CodeGenPrepare/X86/statepoint-relocate.ll | 2 +- .../CodeGenPrepare/X86/tailcall-assume-xbb.ll | 2 +- .../X86/vec-shift-inseltpoison.ll | 14 +- .../CodeGenPrepare/X86/vec-shift.ll | 14 +- .../CodeGenPrepare/X86/widenable-condition.ll | 2 +- .../X86/x86-shuffle-sink-inseltpoison.ll | 8 +- .../CodeGenPrepare/X86/x86-shuffle-sink.ll | 8 +- .../CodeGenPrepare/dead-allocation.ll | 2 +- .../CodeGenPrepare/skip-merging-case-block.ll | 2 +- .../Transforms/HotColdSplit/coldentrycount.ll | 2 +- .../codegenprepare-produced-address-math.ll | 2 +- .../section-accurate-samplepgo.ll | 6 +- llvm/tools/opt/opt.cpp | 2 +- 114 files changed, 240 insertions(+), 402 deletions(-) delete mode 100644 llvm/include/llvm/CodeGen/CodeGenPrepare.h diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h index bba675f1d3eb..dfb8d5d9f2f5 100644 --- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h @@ -21,14 +21,11 @@ #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/IR/Module.h" -#include "llvm/IR/PassManager.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Error.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Target/TargetMachine.h" - using namespace llvm; namespace llvm { @@ -75,13 +72,25 @@ template <> struct DenseMapInfo { } }; -class BasicBlockSectionsProfileReader { +class BasicBlockSectionsProfileReader : public ImmutablePass { public: - friend class BasicBlockSectionsProfileReaderWrapperPass; + static char ID; + BasicBlockSectionsProfileReader(const MemoryBuffer *Buf) - : MBuf(Buf), LineIt(*Buf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'){}; + : ImmutablePass(ID), MBuf(Buf), + LineIt(*Buf, /*SkipBlanks=*/true, /*CommentMarker=*/'#') { + initializeBasicBlockSectionsProfileReaderPass( + *PassRegistry::getPassRegistry()); + }; - BasicBlockSectionsProfileReader(){}; + BasicBlockSectionsProfileReader() : ImmutablePass(ID) { + initializeBasicBlockSectionsProfileReaderPass( + *PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { + return "Basic Block Sections Profile Reader"; + } // Returns true if basic block sections profile exist for function \p // FuncName. @@ -100,6 +109,10 @@ public: SmallVector> getClonePathsForFunction(StringRef FuncName) const; + // Initializes the FunctionNameToDIFilename map for the current module and + // then reads the profile for the matching functions. + bool doInitialization(Module &M) override; + private: StringRef getAliasName(StringRef FuncName) const { auto R = FuncAliasMap.find(FuncName); @@ -157,61 +170,7 @@ private: // sections profile. \p Buf is a memory buffer that contains the list of // functions and basic block ids to selectively enable basic block sections. ImmutablePass * -createBasicBlockSectionsProfileReaderWrapperPass(const MemoryBuffer *Buf); - -/// Analysis pass providing the \c BasicBlockSectionsProfileReader. -/// -/// Note that this pass's result cannot be invalidated, it is immutable for the -/// life of the module. -class BasicBlockSectionsProfileReaderAnalysis - : public AnalysisInfoMixin { - -public: - static AnalysisKey Key; - typedef BasicBlockSectionsProfileReader Result; - BasicBlockSectionsProfileReaderAnalysis(const TargetMachine *TM) : TM(TM) {} - - Result run(Function &F, FunctionAnalysisManager &AM); - -private: - const TargetMachine *TM; -}; - -class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass { -public: - static char ID; - BasicBlockSectionsProfileReader BBSPR; - - BasicBlockSectionsProfileReaderWrapperPass(const MemoryBuffer *Buf) - : ImmutablePass(ID), BBSPR(BasicBlockSectionsProfileReader(Buf)) { - initializeBasicBlockSectionsProfileReaderWrapperPassPass( - *PassRegistry::getPassRegistry()); - }; - - BasicBlockSectionsProfileReaderWrapperPass() - : ImmutablePass(ID), BBSPR(BasicBlockSectionsProfileReader()) { - initializeBasicBlockSectionsProfileReaderWrapperPassPass( - *PassRegistry::getPassRegistry()); - } - - StringRef getPassName() const override { - return "Basic Block Sections Profile Reader"; - } - - bool isFunctionHot(StringRef FuncName) const; - - std::pair> - getClusterInfoForFunction(StringRef FuncName) const; - - SmallVector> - getClonePathsForFunction(StringRef FuncName) const; - - // Initializes the FunctionNameToDIFilename map for the current module and - // then reads the profile for the matching functions. - bool doInitialization(Module &M) override; - - BasicBlockSectionsProfileReader &getBBSPR(); -}; +createBasicBlockSectionsProfileReaderPass(const MemoryBuffer *Buf); } // namespace llvm #endif // LLVM_CODEGEN_BASICBLOCKSECTIONSPROFILEREADER_H diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index fc3da3491412..a7cbb0910baa 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -24,7 +24,6 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/CallBrPrepare.h" -#include "llvm/CodeGen/CodeGenPrepare.h" #include "llvm/CodeGen/DwarfEHPrepare.h" #include "llvm/CodeGen/ExpandMemCmp.h" #include "llvm/CodeGen/ExpandReductions.h" @@ -730,7 +729,7 @@ void CodeGenPassBuilder::addPassesToHandleExceptions( template void CodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const { if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP) - addPass(CodeGenPreparePass(&TM)); + addPass(CodeGenPreparePass()); // TODO: Default ctor'd RewriteSymbolPass is no-op. // addPass(RewriteSymbolPass()); } diff --git a/llvm/include/llvm/CodeGen/CodeGenPrepare.h b/llvm/include/llvm/CodeGen/CodeGenPrepare.h deleted file mode 100644 index dee3a9ee53d7..000000000000 --- a/llvm/include/llvm/CodeGen/CodeGenPrepare.h +++ /dev/null @@ -1,35 +0,0 @@ -//===- CodeGenPrepare.h -----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// \file -/// -/// Defines an IR pass for CodeGen Prepare. -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_PREPARE_H -#define LLVM_CODEGEN_PREPARE_H - -#include "llvm/IR/PassManager.h" - -namespace llvm { - -class Function; -class TargetMachine; - -class CodeGenPreparePass : public PassInfoMixin { -private: - const TargetMachine *TM; - -public: - CodeGenPreparePass(const TargetMachine *TM) : TM(TM) {} - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); -}; - -} // end namespace llvm - -#endif // LLVM_CODEGEN_PREPARE_H diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index bf88a43b8a0c..f950dfae7e33 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -42,7 +42,6 @@ FUNCTION_ANALYSIS("targetir", TargetIRAnalysis, #endif FUNCTION_PASS("callbrprepare", CallBrPreparePass, ()) FUNCTION_PASS("cfguard", CFGuardPass, ()) -FUNCTION_PASS("codegenprepare", CodeGenPreparePass, (TM)) FUNCTION_PASS("consthoist", ConstantHoistingPass, ()) FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass, (TM)) FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false)) @@ -132,6 +131,7 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) #endif DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ()) +DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ()) DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ()) DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ()) diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index bbfb8a0dbe26..ca9fbb1def76 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -93,9 +93,9 @@ namespace llvm { MachineFunctionPass *createResetMachineFunctionPass(bool EmitFallbackDiag, bool AbortOnFailedISel); - /// createCodeGenPrepareLegacyPass - Transform the code to expose more pattern + /// createCodeGenPreparePass - Transform the code to expose more pattern /// matching during instruction selection. - FunctionPass *createCodeGenPrepareLegacyPass(); + FunctionPass *createCodeGenPreparePass(); /// This pass implements generation of target-specific intrinsics to support /// handling of complex number arithmetic diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 3db639a68724..46b1e95c3c15 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -54,7 +54,7 @@ void initializeAssignmentTrackingAnalysisPass(PassRegistry &); void initializeAssumptionCacheTrackerPass(PassRegistry&); void initializeAtomicExpandPass(PassRegistry&); void initializeBasicBlockPathCloningPass(PassRegistry &); -void initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &); +void initializeBasicBlockSectionsProfileReaderPass(PassRegistry &); void initializeBasicBlockSectionsPass(PassRegistry &); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAAWrapperPassPass(PassRegistry&); @@ -75,7 +75,7 @@ void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); void initializeCallGraphWrapperPassPass(PassRegistry&); void initializeCheckDebugMachineModulePass(PassRegistry &); -void initializeCodeGenPrepareLegacyPassPass(PassRegistry &); +void initializeCodeGenPreparePass(PassRegistry&); void initializeComplexDeinterleavingLegacyPassPass(PassRegistry&); void initializeConstantHoistingLegacyPassPass(PassRegistry&); void initializeCycleInfoWrapperPassPass(PassRegistry &); diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index fe7fedad18bc..7a21876e565a 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -113,7 +113,7 @@ namespace { (void) llvm::createTailCallEliminationPass(); (void)llvm::createTLSVariableHoistPass(); (void) llvm::createConstantHoistingPass(); - (void)llvm::createCodeGenPrepareLegacyPass(); + (void) llvm::createCodeGenPreparePass(); (void) llvm::createEarlyCSEPass(); (void) llvm::createGVNPass(); (void) llvm::createPostDomTree(); diff --git a/llvm/lib/CodeGen/BasicBlockPathCloning.cpp b/llvm/lib/CodeGen/BasicBlockPathCloning.cpp index 901542e8507b..5d5f3c3da481 100644 --- a/llvm/lib/CodeGen/BasicBlockPathCloning.cpp +++ b/llvm/lib/CodeGen/BasicBlockPathCloning.cpp @@ -196,7 +196,7 @@ class BasicBlockPathCloning : public MachineFunctionPass { public: static char ID; - BasicBlockSectionsProfileReaderWrapperPass *BBSectionsProfileReader = nullptr; + BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr; BasicBlockPathCloning() : MachineFunctionPass(ID) { initializeBasicBlockPathCloningPass(*PassRegistry::getPassRegistry()); @@ -218,7 +218,7 @@ INITIALIZE_PASS_BEGIN( BasicBlockPathCloning, "bb-path-cloning", "Applies path clonings for the -basic-block-sections=list option", false, false) -INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass) +INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReader) INITIALIZE_PASS_END( BasicBlockPathCloning, "bb-path-cloning", "Applies path clonings for the -basic-block-sections=list option", false, @@ -230,14 +230,13 @@ bool BasicBlockPathCloning::runOnMachineFunction(MachineFunction &MF) { if (hasInstrProfHashMismatch(MF)) return false; - return ApplyCloning(MF, - getAnalysis() - .getClonePathsForFunction(MF.getName())); + return ApplyCloning(MF, getAnalysis() + .getClonePathsForFunction(MF.getName())); } void BasicBlockPathCloning::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); - AU.addRequired(); + AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp index 94b5a503fbd0..42997d2287d6 100644 --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -103,7 +103,7 @@ class BasicBlockSections : public MachineFunctionPass { public: static char ID; - BasicBlockSectionsProfileReaderWrapperPass *BBSectionsProfileReader = nullptr; + BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr; BasicBlockSections() : MachineFunctionPass(ID) { initializeBasicBlockSectionsPass(*PassRegistry::getPassRegistry()); @@ -128,7 +128,7 @@ INITIALIZE_PASS_BEGIN( "Prepares for basic block sections, by splitting functions " "into clusters of basic blocks.", false, false) -INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass) +INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReader) INITIALIZE_PASS_END(BasicBlockSections, "bbsections-prepare", "Prepares for basic block sections, by splitting functions " "into clusters of basic blocks.", @@ -306,7 +306,7 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) { DenseMap FuncClusterInfo; if (BBSectionsType == BasicBlockSection::List) { auto [HasProfile, ClusterInfo] = - getAnalysis() + getAnalysis() .getClusterInfoForFunction(MF.getName()); if (!HasProfile) return false; @@ -362,7 +362,7 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) { void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); - AU.addRequired(); + AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp index 79e42d9304df..15b6f63e8632 100644 --- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp +++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp @@ -30,9 +30,8 @@ using namespace llvm; -char BasicBlockSectionsProfileReaderWrapperPass::ID = 0; -INITIALIZE_PASS(BasicBlockSectionsProfileReaderWrapperPass, - "bbsections-profile-reader", +char BasicBlockSectionsProfileReader::ID = 0; +INITIALIZE_PASS(BasicBlockSectionsProfileReader, "bbsections-profile-reader", "Reads and parses a basic block sections profile.", false, false) @@ -396,11 +395,11 @@ Error BasicBlockSectionsProfileReader::ReadProfile() { } } -bool BasicBlockSectionsProfileReaderWrapperPass::doInitialization(Module &M) { - if (!BBSPR.MBuf) +bool BasicBlockSectionsProfileReader::doInitialization(Module &M) { + if (!MBuf) return false; // Get the function name to debug info filename mapping. - BBSPR.FunctionNameToDIFilename.clear(); + FunctionNameToDIFilename.clear(); for (const Function &F : M) { SmallString<128> DIFilename; if (F.isDeclaration()) @@ -412,46 +411,15 @@ bool BasicBlockSectionsProfileReaderWrapperPass::doInitialization(Module &M) { DIFilename = sys::path::remove_leading_dotslash(CU->getFilename()); } [[maybe_unused]] bool inserted = - BBSPR.FunctionNameToDIFilename.try_emplace(F.getName(), DIFilename) - .second; + FunctionNameToDIFilename.try_emplace(F.getName(), DIFilename).second; assert(inserted); } - if (auto Err = BBSPR.ReadProfile()) + if (auto Err = ReadProfile()) report_fatal_error(std::move(Err)); return false; } -AnalysisKey BasicBlockSectionsProfileReaderAnalysis::Key; - -BasicBlockSectionsProfileReader -BasicBlockSectionsProfileReaderAnalysis::run(Function &F, - FunctionAnalysisManager &AM) { - return BasicBlockSectionsProfileReader(TM->getBBSectionsFuncListBuf()); -} - -bool BasicBlockSectionsProfileReaderWrapperPass::isFunctionHot( - StringRef FuncName) const { - return BBSPR.isFunctionHot(FuncName); -} - -std::pair> -BasicBlockSectionsProfileReaderWrapperPass::getClusterInfoForFunction( - StringRef FuncName) const { - return BBSPR.getClusterInfoForFunction(FuncName); -} - -SmallVector> -BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction( - StringRef FuncName) const { - return BBSPR.getClonePathsForFunction(FuncName); -} - -BasicBlockSectionsProfileReader & -BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() { - return BBSPR; -} - -ImmutablePass *llvm::createBasicBlockSectionsProfileReaderWrapperPass( - const MemoryBuffer *Buf) { - return new BasicBlockSectionsProfileReaderWrapperPass(Buf); +ImmutablePass * +llvm::createBasicBlockSectionsProfileReaderPass(const MemoryBuffer *Buf) { + return new BasicBlockSectionsProfileReader(Buf); } diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp index 418066452c17..7b73a7b11ddf 100644 --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -30,7 +30,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeCFIFixupPass(Registry); initializeCFIInstrInserterPass(Registry); initializeCheckDebugMachineModulePass(Registry); - initializeCodeGenPrepareLegacyPassPass(Registry); + initializeCodeGenPreparePass(Registry); initializeDeadMachineInstructionElimPass(Registry); initializeDebugifyMachineModulePass(Registry); initializeDetectDeadLanesPass(Registry); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index 35a70f0b8c23..5bd4c6b067d7 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/CodeGenPrepare.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -302,8 +301,7 @@ using ValueToSExts = MapVector; class TypePromotionTransaction; -class CodeGenPrepare { - friend class CodeGenPrepareLegacyPass; +class CodeGenPrepare : public FunctionPass { const TargetMachine *TM = nullptr; const TargetSubtargetInfo *SubtargetInfo = nullptr; const TargetLowering *TLI = nullptr; @@ -367,8 +365,6 @@ class CodeGenPrepare { std::unique_ptr DT; public: - CodeGenPrepare(){}; - CodeGenPrepare(const TargetMachine *TM) : TM(TM){}; /// If encounter huge function, we need to limit the build time. bool IsHugeFunc = false; @@ -378,7 +374,15 @@ public: /// to insert such BB into FreshBBs for huge function. SmallSet FreshBBs; - void releaseMemory() { + static char ID; // Pass identification, replacement for typeid + + CodeGenPrepare() : FunctionPass(ID) { + initializeCodeGenPreparePass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; + + void releaseMemory() override { // Clear per function information. InsertedInsts.clear(); PromotedInsts.clear(); @@ -387,7 +391,17 @@ public: BFI.reset(); } - bool run(Function &F, FunctionAnalysisManager &AM); + StringRef getPassName() const override { return "CodeGen Prepare"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + // FIXME: When we can selectively preserve passes, preserve the domtree. + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + AU.addUsedIfAvailable(); + } private: template @@ -474,107 +488,45 @@ private: bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT); bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT); void verifyBFIUpdates(Function &F); - bool _run(Function &F); -}; - -class CodeGenPrepareLegacyPass : public FunctionPass { -public: - static char ID; // Pass identification, replacement for typeid - - CodeGenPrepareLegacyPass() : FunctionPass(ID) { - initializeCodeGenPrepareLegacyPassPass(*PassRegistry::getPassRegistry()); - } - - bool runOnFunction(Function &F) override; - - StringRef getPassName() const override { return "CodeGen Prepare"; } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - // FIXME: When we can selectively preserve passes, preserve the domtree. - AU.addRequired(); - AU.addRequired(); - AU.addRequired(); - AU.addRequired(); - AU.addRequired(); - AU.addUsedIfAvailable(); - } }; } // end anonymous namespace -char CodeGenPrepareLegacyPass::ID = 0; +char CodeGenPrepare::ID = 0; -bool CodeGenPrepareLegacyPass::runOnFunction(Function &F) { - if (skipFunction(F)) - return false; - auto TM = &getAnalysis().getTM(); - CodeGenPrepare CGP(TM); - CGP.DL = &F.getParent()->getDataLayout(); - CGP.SubtargetInfo = TM->getSubtargetImpl(F); - CGP.TLI = CGP.SubtargetInfo->getTargetLowering(); - CGP.TRI = CGP.SubtargetInfo->getRegisterInfo(); - CGP.TLInfo = &getAnalysis().getTLI(F); - CGP.TTI = &getAnalysis().getTTI(F); - CGP.LI = &getAnalysis().getLoopInfo(); - CGP.BPI.reset(new BranchProbabilityInfo(F, *CGP.LI)); - CGP.BFI.reset(new BlockFrequencyInfo(F, *CGP.BPI, *CGP.LI)); - CGP.PSI = &getAnalysis().getPSI(); - auto BBSPRWP = - getAnalysisIfAvailable(); - CGP.BBSectionsProfileReader = BBSPRWP ? &BBSPRWP->getBBSPR() : nullptr; - - return CGP._run(F); -} - -INITIALIZE_PASS_BEGIN(CodeGenPrepareLegacyPass, DEBUG_TYPE, +INITIALIZE_PASS_BEGIN(CodeGenPrepare, DEBUG_TYPE, "Optimize for code generation", false, false) -INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass) +INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReader) INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) -INITIALIZE_PASS_END(CodeGenPrepareLegacyPass, DEBUG_TYPE, - "Optimize for code generation", false, false) +INITIALIZE_PASS_END(CodeGenPrepare, DEBUG_TYPE, "Optimize for code generation", + false, false) -FunctionPass *llvm::createCodeGenPrepareLegacyPass() { - return new CodeGenPrepareLegacyPass(); -} +FunctionPass *llvm::createCodeGenPreparePass() { return new CodeGenPrepare(); } -PreservedAnalyses CodeGenPreparePass::run(Function &F, - FunctionAnalysisManager &AM) { - CodeGenPrepare CGP(TM); +bool CodeGenPrepare::runOnFunction(Function &F) { + if (skipFunction(F)) + return false; - bool Changed = CGP.run(F, AM); - if (!Changed) - return PreservedAnalyses::all(); + DL = &F.getParent()->getDataLayout(); - PreservedAnalyses PA; - PA.preserveSet(); - PA.preserve(); - return PA; -} + bool EverMadeChange = false; -bool CodeGenPrepare::run(Function &F, FunctionAnalysisManager &AM) { - DL = &F.getParent()->getDataLayout(); + TM = &getAnalysis().getTM(); SubtargetInfo = TM->getSubtargetImpl(F); TLI = SubtargetInfo->getTargetLowering(); TRI = SubtargetInfo->getRegisterInfo(); - TLInfo = &AM.getResult(F); - TTI = &AM.getResult(F); - LI = &AM.getResult(F); + TLInfo = &getAnalysis().getTLI(F); + TTI = &getAnalysis().getTTI(F); + LI = &getAnalysis().getLoopInfo(); BPI.reset(new BranchProbabilityInfo(F, *LI)); BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); - auto &MAMProxy = AM.getResult(F); - PSI = MAMProxy.getCachedResult(*F.getParent()); + PSI = &getAnalysis().getPSI(); BBSectionsProfileReader = - AM.getCachedResult(F); - return _run(F); -} - -bool CodeGenPrepare::_run(Function &F) { - bool EverMadeChange = false; - + getAnalysisIfAvailable(); OptSize = F.hasOptSize(); // Use the basic-block-sections profile to promote hot functions to .text.hot // if requested. diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index 3bbc792f4cbf..4003a08a5422 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -978,7 +978,7 @@ void TargetPassConfig::addPassesToHandleExceptions() { /// before exception handling preparation passes. void TargetPassConfig::addCodeGenPrepare() { if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP) - addPass(createCodeGenPrepareLegacyPass()); + addPass(createCodeGenPreparePass()); } /// Add common passes that perform LLVM IR to IR transforms in preparation for @@ -1271,7 +1271,7 @@ void TargetPassConfig::addMachinePasses() { // together. Update this check once we have addressed any issues. if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) { if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) { - addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass( + addPass(llvm::createBasicBlockSectionsProfileReaderPass( TM->getBBSectionsFuncListBuf())); addPass(llvm::createBasicBlockPathCloningPass()); } diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index ca943b43404f..439f749bda8b 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -72,9 +72,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/Analysis/UniformityAnalysis.h" -#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h" #include "llvm/CodeGen/CallBrPrepare.h" -#include "llvm/CodeGen/CodeGenPrepare.h" #include "llvm/CodeGen/DwarfEHPrepare.h" #include "llvm/CodeGen/ExpandLargeDivRem.h" #include "llvm/CodeGen/ExpandLargeFpConvert.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index a8965335b2ea..82ce040c6496 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -229,7 +229,6 @@ CGSCC_PASS_WITH_PARAMS( FUNCTION_ANALYSIS("aa", AAManager()) FUNCTION_ANALYSIS("access-info", LoopAccessAnalysis()) FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis()) -FUNCTION_ANALYSIS("bb-sections-profile-reader", BasicBlockSectionsProfileReaderAnalysis(TM)) FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis()) FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis()) FUNCTION_ANALYSIS("cycles", CycleAnalysis()) @@ -291,7 +290,6 @@ FUNCTION_PASS("break-crit-edges", BreakCriticalEdgesPass()) FUNCTION_PASS("callbrprepare", CallBrPreparePass()) FUNCTION_PASS("callsite-splitting", CallSiteSplittingPass()) FUNCTION_PASS("chr", ControlHeightReductionPass()) -FUNCTION_PASS("codegenprepare", CodeGenPreparePass(TM)) FUNCTION_PASS("consthoist", ConstantHoistingPass()) FUNCTION_PASS("constraint-elimination", ConstraintEliminationPass()) FUNCTION_PASS("coro-elide", CoroElidePass()) diff --git a/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll b/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll index 10b594978bee..92f29dac13fa 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-codegen-prepare-atp.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' < %s -S | FileCheck %s +; RUN: opt -codegenprepare < %s -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll index 4d085869de24..f4e9551259e4 100644 --- a/llvm/test/CodeGen/AArch64/and-sink.ll +++ b/llvm/test/CodeGen/AArch64/and-sink.ll @@ -1,6 +1,6 @@ ; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs < %s | FileCheck %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s -; RUN: opt -S -passes='require,function(codegenprepare)' -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s +; RUN: opt -S -codegenprepare -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s +; RUN: opt -S -codegenprepare -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s @A = dso_local global i32 zeroinitializer @B = dso_local global i32 zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll index 2b42a3f29a72..6041904dc0f3 100644 --- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll +++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s +; RUN: opt -codegenprepare -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s ; RUN: llc < %s -mtriple=arm64-eabi | FileCheck --check-prefix=LLC %s %struct.X = type { i8, i8, [2 x i8] } diff --git a/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll b/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll index 78a0bb4982b6..23cbad0d15b4 100644 --- a/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll +++ b/llvm/test/CodeGen/AArch64/arm64-codegen-prepare-extload.ll @@ -1,6 +1,6 @@ -; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-apple-ios -S | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS -; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-apple-ios -S -stress-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS -; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-apple-ios -S -disable-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=DISABLE +; RUN: opt -codegenprepare < %s -mtriple=aarch64-apple-ios -S | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS +; RUN: opt -codegenprepare < %s -mtriple=aarch64-apple-ios -S -stress-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS +; RUN: opt -codegenprepare < %s -mtriple=aarch64-apple-ios -S -disable-cgp-ext-ld-promotion | FileCheck -enable-var-scope %s --check-prefix=OPTALL --check-prefix=DISABLE ; CodeGenPrepare should move the zext into the block with the load ; so that SelectionDAG can select it with the load. diff --git a/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll b/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll index c33159163901..964c669439d4 100644 --- a/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll +++ b/llvm/test/CodeGen/AArch64/arm64_32-gep-sink.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=arm64_32-apple-ios %s -S -o - | FileCheck %s +; RUN: opt -codegenprepare -mtriple=arm64_32-apple-ios %s -S -o - | FileCheck %s define void @test_simple_sink(ptr %base, i64 %offset) { ; CHECK-LABEL: define void @test_simple_sink( diff --git a/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll b/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll index dfcedc8dd984..98b820709e82 100644 --- a/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll +++ b/llvm/test/CodeGen/AArch64/cgp-trivial-phi-node.ll @@ -1,5 +1,5 @@ ; Checks that case when GEP is bound to trivial PHI node is correctly handled. -; RUN: opt %s -mtriple=aarch64-linux-gnu -passes='require,function(codegenprepare)' -S -o - | FileCheck %s +; RUN: opt %s -mtriple=aarch64-linux-gnu -codegenprepare -S -o - | FileCheck %s ; CHECK: define void @crash(ptr %s, i32 %n) { ; CHECK-NEXT: entry: diff --git a/llvm/test/CodeGen/AArch64/convertphitype.ll b/llvm/test/CodeGen/AArch64/convertphitype.ll index b723b470266a..a5fc46d2abca 100644 --- a/llvm/test/CodeGen/AArch64/convertphitype.ll +++ b/llvm/test/CodeGen/AArch64/convertphitype.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -cgp-optimize-phi-types %s -S | FileCheck %s +; RUN: opt -codegenprepare -cgp-optimize-phi-types %s -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll b/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll index a45f3733dbda..e6ab52dc9e61 100644 --- a/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll +++ b/llvm/test/CodeGen/AArch64/scalable-vector-promotion.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -mtriple=aarch64 -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -mtriple=aarch64 -codegenprepare -S < %s | FileCheck %s ; This test intends to check vector promotion for scalable vector. Current target lowering ; rejects scalable vector before reaching getConstantVector() in CodeGenPrepare. This test diff --git a/llvm/test/CodeGen/AArch64/sve-vscale.ll b/llvm/test/CodeGen/AArch64/sve-vscale.ll index 7214c98cc318..fa48808ff7f8 100644 --- a/llvm/test/CodeGen/AArch64/sve-vscale.ll +++ b/llvm/test/CodeGen/AArch64/sve-vscale.ll @@ -1,5 +1,5 @@ ; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=0 < %s | FileCheck %s -; RUN: opt -mtriple=aarch64 -passes='require,function(codegenprepare)' -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s +; RUN: opt -mtriple=aarch64 -codegenprepare -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s ; ; RDVL diff --git a/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll b/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll index bf3082d99c66..c80aa82ef9a8 100644 --- a/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll +++ b/llvm/test/CodeGen/AArch64/sve2-vscale-sinking.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt -passes='require,function(codegenprepare)' -S -o - %s | FileCheck %s +; RUN: opt -codegenprepare -S -o - %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll index 88a8e7cc4220..3005b17e0524 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck --check-prefixes=OPT,OPT-GFX7 %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tonga < %s | FileCheck --check-prefixes=OPT,OPT-GFX8 %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck --check-prefixes=OPT,OPT-GFX9 %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=gfx1030 < %s | FileCheck --check-prefixes=OPT,OPT-GFX10 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck --check-prefixes=OPT,OPT-GFX7 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga < %s | FileCheck --check-prefixes=OPT,OPT-GFX8 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck --check-prefixes=OPT,OPT-GFX9 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx1030 < %s | FileCheck --check-prefixes=OPT,OPT-GFX10 %s ; RUN: llc -march=amdgcn -mcpu=bonaire < %s | FileCheck --check-prefix=GFX7 %s ; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=GFX8 %s diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll index 1588dde19cfb..aee6f0e82d25 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=OPT %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=OPT %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GCN %s ; Make sure we match the addressing mode offset of csub intrinsics across blocks. diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll index ac50fb86c96f..494b4b5c48ba 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx908.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=OPT %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=OPT %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck -check-prefix=GCN %s ; Make sure we match the addressing mode offset of globla.atomic.fadd intrinsics across blocks. diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll index 65a604c98c4b..0e61547c27b4 100644 --- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll +++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll @@ -1,7 +1,7 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI -check-prefix=OPT-SICIVI %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI -check-prefix=OPT-SICIVI %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI -check-prefix=OPT-SICIVI %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-GFX9 %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI -check-prefix=OPT-SICIVI %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI -check-prefix=OPT-SICIVI %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI -check-prefix=OPT-SICIVI %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-GFX9 %s ; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SICIVI %s ; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=SICIVI %s ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s diff --git a/llvm/test/CodeGen/ARM/vector-promotion.ll b/llvm/test/CodeGen/ARM/vector-promotion.ll index f4a2a4a4521e..3e314306ff08 100644 --- a/llvm/test/CodeGen/ARM/vector-promotion.ll +++ b/llvm/test/CodeGen/ARM/vector-promotion.ll @@ -1,5 +1,5 @@ -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-NORMAL %s -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S -stress-cgp-store-extract | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-STRESS %s +; RUN: opt -codegenprepare -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-NORMAL %s +; RUN: opt -codegenprepare -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon -S -stress-cgp-store-extract | FileCheck --check-prefix=IR-BOTH --check-prefix=IR-STRESS %s ; RUN: llc -mtriple=thumbv7-apple-ios %s -o - -mattr=+neon | FileCheck --check-prefix=ASM %s ; IR-BOTH-LABEL: @simpleOneInstructionPromotion diff --git a/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll b/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll index 2a8d05c4f26e..b02bdc3b5724 100644 --- a/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll +++ b/llvm/test/CodeGen/Generic/addr-sink-call-multi-arg.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s ; REQUIRES: aarch64-registered-target ; Check that we don't give up if unable to sink the first argument. diff --git a/llvm/test/CodeGen/Generic/addr-use-count.ll b/llvm/test/CodeGen/Generic/addr-use-count.ll index 5c4c0c618794..00943b5a58e2 100644 --- a/llvm/test/CodeGen/Generic/addr-use-count.ll +++ b/llvm/test/CodeGen/Generic/addr-use-count.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s ; REQUIRES: aarch64-registered-target ; Test that `%addr` is sunk, after we've increased limit on the number of the memory uses to scan. diff --git a/llvm/test/CodeGen/X86/callbr-codegenprepare.ll b/llvm/test/CodeGen/X86/callbr-codegenprepare.ll index d2380a730b5b..854cc4bf7a9c 100644 --- a/llvm/test/CodeGen/X86/callbr-codegenprepare.ll +++ b/llvm/test/CodeGen/X86/callbr-codegenprepare.ll @@ -1,4 +1,4 @@ -;; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +;; RUN: opt -S -codegenprepare < %s | FileCheck %s ;; Ensure that codegenprepare (via InstSimplify) doesn't eliminate the ;; phi here (which would cause a module verification error). diff --git a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll index c611e89f2786..6e95c91e7398 100644 --- a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll +++ b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-sext.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' %s -o - | FileCheck %s +; RUN: opt -S -codegenprepare %s -o - | FileCheck %s ; This file tests the different cases what are involved when codegen prepare ; tries to get sign/zero extension out of the way of addressing mode. ; This tests require an actual target as addressing mode decisions depends diff --git a/llvm/test/CodeGen/X86/codegen-prepare-extload.ll b/llvm/test/CodeGen/X86/codegen-prepare-extload.ll index ce87985fb3a0..ff0ac5bfd6a7 100644 --- a/llvm/test/CodeGen/X86/codegen-prepare-extload.ll +++ b/llvm/test/CodeGen/X86/codegen-prepare-extload.ll @@ -1,9 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s ; RUN: llc < %s -mtriple=x86_64-win64 | FileCheck %s -; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=x86_64-apple-macosx -S | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS -; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=x86_64-apple-macosx -S -stress-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS -; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=x86_64-apple-macosx -S -disable-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=DISABLE +; RUN: opt -codegenprepare < %s -mtriple=x86_64-apple-macosx -S | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=NONSTRESS +; RUN: opt -codegenprepare < %s -mtriple=x86_64-apple-macosx -S -stress-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=OPT --check-prefix=STRESS +; RUN: opt -codegenprepare < %s -mtriple=x86_64-apple-macosx -S -disable-cgp-ext-ld-promotion | FileCheck %s --check-prefix=OPTALL --check-prefix=DISABLE ; rdar://7304838 ; CodeGenPrepare should move the zext into the block with the load diff --git a/llvm/test/CodeGen/X86/convertphitype.ll b/llvm/test/CodeGen/X86/convertphitype.ll index 6c77236fc698..df01612252bf 100644 --- a/llvm/test/CodeGen/X86/convertphitype.ll +++ b/llvm/test/CodeGen/X86/convertphitype.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -cgp-optimize-phi-types=true %s -S | FileCheck %s +; RUN: opt -codegenprepare -cgp-optimize-phi-types=true %s -S | FileCheck %s target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128" target triple = "i386-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll b/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll index ac23de49d3bd..6b953e300425 100644 --- a/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll +++ b/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S -passes='require,function(codegenprepare)' %s -o - | FileCheck %s +; RUN: opt -S -codegenprepare %s -o - | FileCheck %s target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/X86/pr58538.ll b/llvm/test/CodeGen/X86/pr58538.ll index 7bae2594fc02..6c4103718950 100644 --- a/llvm/test/CodeGen/X86/pr58538.ll +++ b/llvm/test/CodeGen/X86/pr58538.ll @@ -1,5 +1,5 @@ -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64 %s -S -o - | FileCheck %s -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=i386 %s -S -o - | FileCheck %s +; RUN: opt -codegenprepare -mtriple=x86_64 %s -S -o - | FileCheck %s +; RUN: opt -codegenprepare -mtriple=i386 %s -S -o - | FileCheck %s define i32 @f(i32 %0) { ; CHECK-LABEL: @f diff --git a/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll b/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll index 75bbae1050d6..48440558283d 100644 --- a/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll +++ b/llvm/test/CodeGen/X86/tailcall-cgp-dup.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s -; RUN: opt -S -passes='require,function(codegenprepare)' %s -mtriple=x86_64-apple-darwin -o - | FileCheck %s --check-prefix OPT +; RUN: opt -S -codegenprepare %s -mtriple=x86_64-apple-darwin -o - | FileCheck %s --check-prefix OPT ; Teach CGP to dup returns to enable tail call optimization. ; rdar://9147433 diff --git a/llvm/test/CodeGen/X86/tailcall-extract.ll b/llvm/test/CodeGen/X86/tailcall-extract.ll index 7a6c75c44ca7..c3597a8e5b99 100644 --- a/llvm/test/CodeGen/X86/tailcall-extract.ll +++ b/llvm/test/CodeGen/X86/tailcall-extract.ll @@ -1,5 +1,5 @@ ; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s --check-prefix OPT +; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s --check-prefix OPT ; The exit block containing extractvalue can be duplicated into the BB diff --git a/llvm/test/DebugInfo/ARM/salvage-debug-info.ll b/llvm/test/DebugInfo/ARM/salvage-debug-info.ll index 1564a80ded0e..3717abada42e 100644 --- a/llvm/test/DebugInfo/ARM/salvage-debug-info.ll +++ b/llvm/test/DebugInfo/ARM/salvage-debug-info.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S %s -o - | FileCheck %s +; RUN: opt -codegenprepare -S %s -o - | FileCheck %s ; typedef struct info { ; unsigned long long size; ; } info_t; diff --git a/llvm/test/DebugInfo/X86/zextload.ll b/llvm/test/DebugInfo/X86/zextload.ll index 05d92b09c20c..888e230c258d 100644 --- a/llvm/test/DebugInfo/X86/zextload.ll +++ b/llvm/test/DebugInfo/X86/zextload.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s ; ; This test case was generated from the following source code: ; diff --git a/llvm/test/Other/codegenprepare-and-debug.ll b/llvm/test/Other/codegenprepare-and-debug.ll index 95f48c630efb..9023e8f21997 100644 --- a/llvm/test/Other/codegenprepare-and-debug.ll +++ b/llvm/test/Other/codegenprepare-and-debug.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -codegenprepare -S < %s | FileCheck %s ; RUN: opt -strip-debug -codegenprepare -S < %s | FileCheck %s ; REQUIRES: x86-registered-target diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll index 25d4492f4c16..91194864c013 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/combine-address-mode.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s +; RUN: opt -S -codegenprepare -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s @_MergedGlobals = external dso_local global <{ i32, i32 }>, align 4 diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll index de7eeada6024..adb13f1a4c9f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck -enable-var-scope %s +; RUN: opt -S -codegenprepare -mtriple=aarch64-linux %s | FileCheck -enable-var-scope %s ; Test for CodeGenPrepare::optimizeLoadExt(): simple case: two loads ; feeding a phi that zext's each loaded value. diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll index 469d818af28f..0114d7f9f409 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll index 6444f6adcdcc..e4c5b4ceee54 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll index f72679f55e11..4caf6d0dc893 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/overflow-intrinsics.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s -; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG ; Subset of tests from llvm/tests/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll ; to test shouldFormOverflowOp on SPARC, where it is not profitable to create @@ -167,5 +167,5 @@ define i1 @usubo_ult_i64_math_overflow_used(i64 %x, i64 %y, ptr %p) { ret i1 %ov } -; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. +; Check that every instruction inserted by -codegenprepare has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll index f170c8ff18c1..73322836d1b8 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S --passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S --codegenprepare < %s | FileCheck %s target triple = "aarch64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll index fa53d536fa38..6a8b5733889e 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=arm64-apple-ios7.0 %s | FileCheck %s +; RUN: opt -S -codegenprepare -mtriple=arm64-apple-ios7.0 %s | FileCheck %s %foo = type { i8 } diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll index 8999e8d901ad..60b1e81e3dcd 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/zext-to-shuffle.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S %s | FileCheck %s +; RUN: opt -codegenprepare -S %s | FileCheck %s target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" target triple = "arm64-apple-ios" diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll index 20a5a9ededac..acd40d744f71 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/addressing-modes.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn--amdhsa < %s | FileCheck %s define amdgpu_kernel void @test_sink_as999_small_max_mubuf_offset(ptr addrspace(999) %out, ptr addrspace(999) %in) { ; CHECK-LABEL: @test_sink_as999_small_max_mubuf_offset( diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll index c64d5a357f3a..63098ab098a2 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown < %s | FileCheck -check-prefix=ASC -check-prefix=COMMON %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown < %s | FileCheck -check-prefix=ASC -check-prefix=COMMON %s ; COMMON-LABEL: @test_sink_ptrtoint_asc( ; ASC: addrspacecast diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll index 77e79a902f3b..7e6020629df8 100644 --- a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: opt -S -codegenprepare -mtriple=amdgcn--amdhsa < %s | FileCheck %s define i64 @no_sink_local_to_flat(i1 %pred, ptr addrspace(3) %ptr) { ; CHECK-LABEL: define i64 @no_sink_local_to_flat( diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll index ff5cef7e781f..996cab1d1d2c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/branch-on-zero.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv8.1m.main-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll index f84491933b25..52c06fd52b7b 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/dead-gep.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S %s -o - | FileCheck %s +; RUN: opt -codegenprepare -S %s -o - | FileCheck %s target triple = "thumbv7-apple-ios7.0.0" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll index 9bec9b53ed0c..ae76dbda4aa1 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=arm7-unknown-unknown -S < %s | FileCheck %s +; RUN: opt -codegenprepare -mtriple=arm7-unknown-unknown -S < %s | FileCheck %s declare void @llvm.memcpy.p0.p0.i32(ptr, ptr, i32, i1) nounwind declare void @llvm.memmove.p0.p0.i32(ptr, ptr, i32, i1) nounwind diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll index 2f32ef94f82c..3fbc21331410 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -codegenprepare -S < %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv8m.main-arm-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll index 49cffe286a54..838486aa2486 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=thumbv7m -disable-complex-addr-modes=false -addr-sink-new-select=true -addr-sink-new-phis=true < %s | FileCheck %s +; RUN: opt -S -codegenprepare -mtriple=thumbv7m -disable-complex-addr-modes=false -addr-sink-new-select=true -addr-sink-new-phis=true < %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll index 69c8b92a3558..cd2087d94149 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' %s | FileCheck %s +; RUN: opt -S -codegenprepare %s | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv6m-arm-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll index 3f113e6ea163..76b119fe36aa 100644 --- a/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll +++ b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -codegenprepare -S < %s | FileCheck %s target triple = "armv8m.main-none-eabi" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll index b8a45e847afc..94adf1970938 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll index d46aaf51a59c..c571da4411e7 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll index 9ec533c2e653..21e47c614ad0 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll index 463fa0d487a7..424f7c3b0271 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll index af9b5ccd7287..17b0dbf81ac2 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-introduce-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll index 7ee7f0550318..374f30dba508 100644 --- a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64" target triple = "nvptx64-nvidia-cuda" diff --git a/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll index 65177d5ae3d7..79aebb9c247a 100644 --- a/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll +++ b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=powerpc64-unknown-linux-gnu -data-layout="E-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=BE %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=powerpc64le-unknown-linux-gnu -data-layout="e-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=LE %s +; RUN: opt -S -codegenprepare -mtriple=powerpc64-unknown-linux-gnu -data-layout="E-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=BE %s +; RUN: opt -S -codegenprepare -mtriple=powerpc64le-unknown-linux-gnu -data-layout="e-m:e-i64:64-n32:64" -force-split-store < %s | FileCheck --check-prefix=LE %s define void @split_store_align1(float %x, ptr %p) { ; BE-LABEL: @split_store_align1( diff --git a/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll b/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll index 863b0b4ad26f..130401d78171 100644 --- a/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll +++ b/llvm/test/Transforms/CodeGenPrepare/RISCV/and-mask-sink.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv32 %s \ +; RUN: opt -S -codegenprepare -mtriple=riscv32 %s \ ; RUN: | FileCheck --check-prefixes=CHECK,NOZBS %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv32 -mattr=+zbs %s \ +; RUN: opt -S -codegenprepare -mtriple=riscv32 -mattr=+zbs %s \ ; RUN: | FileCheck --check-prefixes=CHECK,ZBS %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv64 %s \ +; RUN: opt -S -codegenprepare -mtriple=riscv64 %s \ ; RUN: | FileCheck --check-prefixes=CHECK,NOZBS %s -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=riscv64 -mattr=zbs %s \ +; RUN: opt -S -codegenprepare -mtriple=riscv64 -mattr=zbs %s \ ; RUN: | FileCheck --check-prefixes=CHECK,ZBS %s @A = global i32 zeroinitializer diff --git a/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll b/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll index 00ad32e96748..c70112e91ebd 100644 --- a/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll +++ b/llvm/test/Transforms/CodeGenPrepare/RISCV/cttz-ctlz.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target triple = "riscv64-unknown-unknown" diff --git a/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll index ec60238cbf92..7525ae14fa35 100644 --- a/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/SPARC/overflow-intrinsics.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s -; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG ; Subset of tests from llvm/tests/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll ; to test shouldFormOverflowOp on SPARC, where it is not profitable to create @@ -119,5 +119,5 @@ define i1 @usubo_ult_i64_math_overflow_used(i64 %x, i64 %y, ptr %p) { ret i1 %ov } -; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. +; Check that every instruction inserted by -codegenprepare has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll b/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll index cb617671827e..614e80e3e89c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll @@ -1,5 +1,5 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s -; RUN: opt -passes='require,function(codegenprepare)' -S < %s --try-experimental-debuginfo-iterators | FileCheck %s +; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -codegenprepare -S < %s --try-experimental-debuginfo-iterators | FileCheck %s ; The following target lines are needed for the test to exercise what it should. ; Without these lines, CodeGenPrepare does not try to sink the bitcasts. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll index bcb7edd6e91b..9eede8cf361b 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash-inseltpoison.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S %s | FileCheck %s +; RUN: opt -codegenprepare -S %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll index 9ce830c39477..7433ff74bab5 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/cgp_shuffle_crash.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S %s | FileCheck %s +; RUN: opt -codegenprepare -S %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll b/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll index c1b919d31e07..fbf294128163 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -codegenprepare -S < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll index 3a3a5327da8d..5f368faf46ee 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll @@ -1,10 +1,10 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s --check-prefix=SLOW -; RUN: opt -S -passes='require,function(codegenprepare)' -mattr=+bmi < %s | FileCheck %s --check-prefix=FAST_TZ -; RUN: opt -S -passes='require,function(codegenprepare)' -mattr=+lzcnt < %s | FileCheck %s --check-prefix=FAST_LZ +; RUN: opt -S -codegenprepare < %s | FileCheck %s --check-prefix=SLOW +; RUN: opt -S -codegenprepare -mattr=+bmi < %s | FileCheck %s --check-prefix=FAST_TZ +; RUN: opt -S -codegenprepare -mattr=+lzcnt < %s | FileCheck %s --check-prefix=FAST_LZ -; RUN: opt -S -enable-debugify -passes='require,function(codegenprepare)' < %s | FileCheck %s --check-prefix=DEBUGINFO -; RUN: opt -S -enable-debugify -passes='require,function(codegenprepare)' --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=DEBUGINFO +; RUN: opt -S -debugify -codegenprepare < %s | FileCheck %s --check-prefix=DEBUGINFO +; RUN: opt -S -debugify -codegenprepare --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=DEBUGINFO target triple = "x86_64-unknown-unknown" target datalayout = "e-n32:64" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll b/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll index e1d99fd932fb..abdf54c6e542 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/delete-assume-dead-code.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s define i32 @test1(ptr %d) nounwind { ; CHECK-LABEL: @test1( diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll index 5349afc18d84..a0814e0a5f20 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -disable-cgp-branch-opts -S < %s | FileCheck %s +; RUN: opt -codegenprepare -disable-cgp-branch-opts -S < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll b/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll index e9ecfd1615d4..c37227f5fa82 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/freeze-brcond.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll index e62ba5d5a7f5..41b1ac2c05fc 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s -; RUN: opt -S -passes='require,function(codegenprepare)' -cgpp-huge-func=0 < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -codegenprepare -cgpp-huge-func=0 < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll index 7899477afdb2..2cf98491acb9 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll index a2ea3f7e36a0..d2eae6954c5d 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/gep-unmerging.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s @exit_addr = constant ptr blockaddress(@gep_unmerging, %exit) @op1_addr = constant ptr blockaddress(@gep_unmerging, %op1) diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll b/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll index a0bac01d1165..3c81a4beb834 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/invariant.group.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s @tmp = global i8 0 diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll b/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll index dbd5e87f2c28..ea07a5fe9bc5 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/masked-gather-struct-gep.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s ; REQUIRES: x86-registered-target target triple = "x86_64-pc-linux" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll b/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll index 9d53855ada79..2f42ad889b42 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/nonintegral.ll @@ -1,5 +1,5 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s -; RUN: opt -S -passes='require,function(codegenprepare)' -addr-sink-using-gep=false < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s +; RUN: opt -S -codegenprepare -addr-sink-using-gep=false < %s | FileCheck %s ; This target data layout is modified to have a non-integral addrspace(1), ; in order to verify that codegenprepare does not try to introduce illegal diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll index aaf3df093468..be651d7eb004 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll index 653f34635648..a324f6f44e5c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s -; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "x86_64-apple-darwin10.0.0" @@ -636,6 +636,6 @@ exit: ret void } -; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. +; Check that every instruction inserted by -codegenprepare has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll index 51fba2229f3c..3ef27c7c950f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-windows-msvc" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll index e9d0806235c9..eec9475a1c48 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true %s | FileCheck %s +; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll index b6296871f575..d75e5632ebb2 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr72046.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=x86_64-unknown-unknown < %s | FileCheck %s +; RUN: opt -S -codegenprepare -mtriple=x86_64-unknown-unknown < %s | FileCheck %s ; Make sure the nneg flag is dropped when lshr and zext are interchanged. define i8 @get(ptr %box, i32 %in) { diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll b/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll index eff88bba3773..0366b7d7e6d2 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/recursively-delete-dead-instructions.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s declare void @llvm.assume(i1 noundef) nounwind willreturn diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll b/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll index 6b7a122b3e26..1d5e6ea0978a 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/remove-assume-block.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -S -codegenprepare -mtriple=x86_64-linux < %s | FileCheck %s ; ; Ensure that blocks that only contain @llvm.assume are removed completely ; during CodeGenPrepare. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll index 08dd77e9e4c3..a0f34a882d30 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s -; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s | FileCheck %s -check-prefix=DEBUG -; RUN: opt -enable-debugify -passes='require,function(codegenprepare)' -S < %s --try-experimental-debuginfo-iterators | FileCheck %s -check-prefix=DEBUG +; RUN: opt -codegenprepare -S < %s | FileCheck %s +; RUN: opt -debugify -codegenprepare -S < %s | FileCheck %s -check-prefix=DEBUG +; RUN: opt -debugify -codegenprepare -S < %s --try-experimental-debuginfo-iterators | FileCheck %s -check-prefix=DEBUG target triple = "x86_64-unknown-unknown" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll index 08e822f7e211..45ddbe76c8f1 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll @@ -1,5 +1,5 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-YES -; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=false -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NO +; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-YES +; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=false -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NO target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll index 7660ee47fdbd..d5e69b9d802e 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll index 076915028aef..336421e4c500 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-select=true %s | FileCheck %s --check-prefix=CHECK +; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-select=true %s | FileCheck %s --check-prefix=CHECK target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll index 6a6b029b67b5..611ef908d706 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' -disable-complex-addr-modes=false -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK +; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll index f75af606eff0..97b11a2e1f1c 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s +; RUN: opt -S -codegenprepare < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll index a760f56b151f..f2e82212d0fa 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -passes='require,function(codegenprepare)' < %s | FileCheck %s -check-prefix=CHECK -check-prefix=GEP +; RUN: opt -S -codegenprepare < %s | FileCheck %s -check-prefix=CHECK -check-prefix=GEP target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll b/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll index 7f7f80910b48..c5d18ff50309 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/split-indirect-loop.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s ; Test that an invalid CFG is not created by splitIndirectCriticalEdges ; transformation when the 'target' block is a loop to itself. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll index 0335da94ea50..3bced480f31a 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-unknown-unknown -force-split-store -S < %s | FileCheck %s +; RUN: opt -codegenprepare -mtriple=x86_64-unknown-unknown -force-split-store -S < %s | FileCheck %s target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" target triple = "i686-w64-windows-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll b/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll index babaa08a959b..a8a6f7baf9b4 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/statepoint-relocate.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S < %s | FileCheck %s +; RUN: opt -codegenprepare -S < %s | FileCheck %s target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll index dd47d5eb6cc4..9dc88a100daa 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64-linux < %s | FileCheck %s +; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s ; The ret instruction can be duplicated into BB case2 even though there is an ; intermediate BB exit1 and call to llvm.assume. diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll index db7d960899ca..557974fcfe54 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll @@ -1,10 +1,10 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG define <4 x i32> @vector_variable_shift_right_v4i32(<4 x i1> %cond, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { ; AVX1-LABEL: @vector_variable_shift_right_v4i32( @@ -409,5 +409,5 @@ exit: declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) #1 -; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. +; Check that every instruction inserted by -codegenprepare has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll index e0f04f77efa0..482e822ea3d8 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift.ll @@ -1,10 +1,10 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP -; RUN: opt -passes='require,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX1 +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX2 +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP +; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG define <4 x i32> @vector_variable_shift_right_v4i32(<4 x i1> %cond, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { ; AVX1-LABEL: @vector_variable_shift_right_v4i32( @@ -409,5 +409,5 @@ exit: declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) #1 -; Check that every instruction inserted by -passes='require,function(codegenprepare)' has a debug location. +; Check that every instruction inserted by -codegenprepare has a debug location. ; DEBUG: CheckModuleDebugify: PASS diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll b/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll index 12230ec689cf..b26876e0e1e2 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/widenable-condition.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='require,function(codegenprepare)' -S -mtriple=x86_64 < %s | FileCheck %s +; RUN: opt -codegenprepare -S -mtriple=x86_64 < %s | FileCheck %s ; Check the idiomatic guard pattern to ensure it's lowered correctly. define void @test_guard(i1 %cond_0) { diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll index ce1b6bd5ae63..72d1672eb4f7 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW +; RUN: opt -S -codegenprepare -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 +; RUN: opt -S -codegenprepare -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP +; RUN: opt -S -codegenprepare -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 +; RUN: opt -S -codegenprepare -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-darwin10.9.0" diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll index 9e82844dfc2f..c14918a6956f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll +++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 -; RUN: opt -S -passes='require,function(codegenprepare)' -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW +; RUN: opt -S -codegenprepare -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2 +; RUN: opt -S -codegenprepare -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP +; RUN: opt -S -codegenprepare -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2 +; RUN: opt -S -codegenprepare -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-darwin10.9.0" diff --git a/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll b/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll index 9550e748da6d..637040a0d56d 100644 --- a/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll +++ b/llvm/test/Transforms/CodeGenPrepare/dead-allocation.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; Eliminate the dead allocation instruction ; REQUIRES: arm-registered-target -; RUN: opt -passes='require,function(codegenprepare)' < %s -S | FileCheck %s +; RUN: opt -codegenprepare < %s -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "armv7--linux-gnueabihf" diff --git a/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll b/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll index d25b9c91aff6..608ad4c0a32f 100644 --- a/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll +++ b/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll @@ -1,5 +1,5 @@ ; REQUIRES: aarch64-registered-target -; RUN: opt -passes='require,function(codegenprepare)' < %s -mtriple=aarch64-none-linux-gnu -S | FileCheck %s +; RUN: opt -codegenprepare < %s -mtriple=aarch64-none-linux-gnu -S | FileCheck %s target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll index 6e5ef1aa2539..1a113ff16188 100644 --- a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll +++ b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll @@ -1,5 +1,5 @@ ; REQUIRES: x86-registered-target -; RUN: opt -passes=hotcoldsplit -hotcoldsplit-threshold=0 < %s | opt -passes='require,function(codegenprepare)' -S | FileCheck %s +; RUN: opt -passes=hotcoldsplit -hotcoldsplit-threshold=0 < %s | opt -codegenprepare -S | FileCheck %s ; Test to ensure that split cold function gets 0 entry count profile ; metadata when compiling with pgo. diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll index a56efe8dd3f3..ff8a804beeb5 100644 --- a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll +++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll @@ -1,4 +1,4 @@ -; RUN: opt -passes='require,function(codegenprepare)' -passes=load-store-vectorizer %s -S -o - | FileCheck %s +; RUN: opt -codegenprepare -load-store-vectorizer %s -S -o - | FileCheck %s ; RUN: opt -passes=load-store-vectorizer %s -S -o - | FileCheck %s ; RUN: opt -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s diff --git a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll index ef2ddbc33cee..a404220056c8 100644 --- a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll +++ b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll @@ -1,7 +1,7 @@ ; REQUIRES: x86-registered-target -; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -passes='require,function(codegenprepare)' | FileCheck %s -; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -passes='require,function(codegenprepare)' -profile-unknown-in-special-section -partial-profile | FileCheck %s --check-prefix=UNKNOWN -; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -profile-sample-accurate -S | opt -S -passes='require,function(codegenprepare)' | FileCheck %s --check-prefix=ACCURATE +; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -codegenprepare | FileCheck %s +; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof | opt -S -codegenprepare -profile-unknown-in-special-section -partial-profile | FileCheck %s --check-prefix=UNKNOWN +; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -profile-sample-accurate -S | opt -S -codegenprepare | FileCheck %s --check-prefix=ACCURATE target triple = "x86_64-pc-linux-gnu" diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp index c649e6ecddc0..b6068513d230 100644 --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -426,7 +426,7 @@ int main(int argc, char **argv) { initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry); initializeSelectOptimizePass(Registry); initializeCallBrPreparePass(Registry); - initializeCodeGenPrepareLegacyPassPass(Registry); + initializeCodeGenPreparePass(Registry); initializeAtomicExpandPass(Registry); initializeWinEHPreparePass(Registry); initializeDwarfEHPrepareLegacyPassPass(Registry); -- GitLab From 3eeed79946124a8f67a89bb950a1e510369dcdf9 Mon Sep 17 00:00:00 2001 From: Younan Zhang Date: Fri, 5 Jan 2024 20:36:41 +0800 Subject: [PATCH 101/728] [clang] Correctly implement CWG 2672 (#75001) --- clang/docs/ReleaseNotes.rst | 3 +- clang/lib/Sema/SemaTemplateInstantiate.cpp | 11 +------- clang/lib/Sema/TreeTransform.h | 7 +++++ clang/test/CXX/drs/dr26xx.cpp | 2 +- .../expr.prim.lambda/default-arguments.cpp | 6 ++-- .../expr.prim/expr.prim.lambda/p11-1y.cpp | 2 -- .../expr/expr.prim/expr.prim.lambda/p23.cpp | 1 - .../expr/expr.prim/expr.prim.lambda/p4.cpp | 3 +- clang/test/CXX/temp/temp.deduct/p9.cpp | 8 ++---- clang/test/SemaCXX/cxx1y-init-captures.cpp | 8 +++--- clang/test/SemaCXX/cxx1z-lambda-star-this.cpp | 4 +-- clang/test/SemaCXX/lambda-expressions.cpp | 4 +-- clang/test/SemaCXX/lambda-pack-expansion.cpp | 1 - clang/test/SemaCXX/vartemplate-lambda.cpp | 1 - .../SemaCXX/warn-unused-lambda-capture.cpp | 28 +++++++++---------- .../SemaTemplate/instantiate-local-class.cpp | 4 +-- 16 files changed, 39 insertions(+), 54 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index cd7d50455187..3c63ad96ca2e 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -723,7 +723,8 @@ Bug Fixes to C++ Support - Clang emits an error on substitution failure within lambda body inside a requires-expression. This fixes: - (`#64138 `_). + (`#64138 `_) and + (`#71684 `_). - Update ``FunctionDeclBitfields.NumFunctionDeclBits``. This fixes: (`#64171 `_). diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp index e1cbdcd72eac..27a4b68fd59a 100644 --- a/clang/lib/Sema/SemaTemplateInstantiate.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp @@ -35,7 +35,6 @@ #include "clang/Sema/Template.h" #include "clang/Sema/TemplateDeduction.h" #include "clang/Sema/TemplateInstCallback.h" -#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TimeProfiler.h" @@ -1142,8 +1141,7 @@ std::optional Sema::isSFINAEContext() const { case CodeSynthesisContext::DeducedTemplateArgumentSubstitution: // We're either substituting explicitly-specified template arguments, // deduced template arguments. SFINAE applies unless we are in a lambda - // expression, see [temp.deduct]p9. - [[fallthrough]]; + // body, see [temp.deduct]p9. case CodeSynthesisContext::ConstraintSubstitution: case CodeSynthesisContext::RequirementInstantiation: case CodeSynthesisContext::RequirementParameterInstantiation: @@ -1445,13 +1443,6 @@ namespace { LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true); Sema::ConstraintEvalRAII RAII(*this); - Sema::CodeSynthesisContext C; - C.Kind = clang::Sema::CodeSynthesisContext::LambdaExpressionSubstitution; - C.PointOfInstantiation = E->getBeginLoc(); - SemaRef.pushCodeSynthesisContext(C); - auto PopCtx = - llvm::make_scope_exit([this] { SemaRef.popCodeSynthesisContext(); }); - ExprResult Result = inherited::TransformLambdaExpr(E); if (Result.isInvalid()) return Result; diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h index 5e9b51845707..c8c5a51bf9f9 100644 --- a/clang/lib/Sema/TreeTransform.h +++ b/clang/lib/Sema/TreeTransform.h @@ -13685,10 +13685,17 @@ TreeTransform::TransformLambdaExpr(LambdaExpr *E) { getSema().PushExpressionEvaluationContext( Sema::ExpressionEvaluationContext::PotentiallyEvaluated); + Sema::CodeSynthesisContext C; + C.Kind = clang::Sema::CodeSynthesisContext::LambdaExpressionSubstitution; + C.PointOfInstantiation = E->getBody()->getBeginLoc(); + getSema().pushCodeSynthesisContext(C); + // Instantiate the body of the lambda expression. StmtResult Body = Invalid ? StmtError() : getDerived().TransformLambdaBody(E, E->getBody()); + getSema().popCodeSynthesisContext(); + // ActOnLambda* will pop the function scope for us. FuncScopeCleanup.disable(); diff --git a/clang/test/CXX/drs/dr26xx.cpp b/clang/test/CXX/drs/dr26xx.cpp index dd4bb1ff6ae2..8a22dbeb98a3 100644 --- a/clang/test/CXX/drs/dr26xx.cpp +++ b/clang/test/CXX/drs/dr26xx.cpp @@ -211,7 +211,7 @@ void f(...); template void bar(T) requires requires { - decltype([]() -> T {})::foo(); + []() -> decltype(T::foo()) {}; }; void bar(...); diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/default-arguments.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/default-arguments.cpp index c5d08ec404a7..72265d77700a 100644 --- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/default-arguments.cpp +++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/default-arguments.cpp @@ -35,8 +35,7 @@ struct NoDefaultCtor { template void defargs_in_template_unused(T t) { auto l1 = [](const T& value = T()) { }; // expected-error{{no matching constructor for initialization of 'NoDefaultCtor'}} \ - // expected-note {{in instantiation of default function argument expression for 'operator()' required here}} \ - // expected-note {{while substituting into a lambda expression here}} + // expected-note {{in instantiation of default function argument expression for 'operator()' required here}} l1(t); } @@ -46,8 +45,7 @@ template void defargs_in_template_unused(NoDefaultCtor); // expected-note{{in i template void defargs_in_template_used() { auto l1 = [](const T& value = T()) { }; // expected-error{{no matching constructor for initialization of 'NoDefaultCtor'}} \ - // expected-note {{in instantiation of default function argument expression for 'operator()' required here}} \ - // expected-note {{while substituting into a lambda expression here}} + // expected-note {{in instantiation of default function argument expression for 'operator()' required here}} l1(); } diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p11-1y.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p11-1y.cpp index ed36a33850ce..8e13cc8a7ff5 100644 --- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p11-1y.cpp +++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p11-1y.cpp @@ -35,7 +35,6 @@ auto init_kind_2 = [ec = ExplicitCopy()] {}; // expected-error {{no matching con template void init_kind_template() { auto init_kind_1 = [ec(T())] {}; auto init_kind_2 = [ec = T()] {}; // expected-error {{no matching constructor}} - // expected-note@-1 {{while substituting into a lambda expression here}} } template void init_kind_template(); template void init_kind_template(); // expected-note {{instantiation of}} @@ -53,7 +52,6 @@ auto bad_init_6 = [a{overload_fn}] {}; // expected-error {{cannot deduce type fo auto bad_init_7 = [a{{1}}] {}; // expected-error {{cannot deduce type for lambda capture 'a' from nested initializer list}} template void pack_1(T...t) { (void)[a(t...)] {}; } // expected-error {{initializer missing for lambda capture 'a'}} - // expected-note@-1 {{while substituting into a lambda expression here}} template void pack_1<>(); // expected-note {{instantiation of}} // No lifetime-extension of the temporary here. diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p23.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p23.cpp index a5278c27bf25..028fcee5fda4 100644 --- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p23.cpp +++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p23.cpp @@ -85,7 +85,6 @@ void init_capture_pack_err(Args ...args) { template void init_capture_pack_multi(Args ...args) { [as(args...)] {} (); // expected-error {{initializer missing for lambda capture 'as'}} expected-error {{multiple}} - // expected-note@-1 2{{while substituting into a lambda expression}} } template void init_capture_pack_multi(); // expected-note {{instantiation}} template void init_capture_pack_multi(int); diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp index d933f7650043..e95a0965a2a3 100644 --- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp +++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp @@ -55,12 +55,11 @@ void test_result_type(int N) { // expected-note {{declared here}} template void test_result_type_tpl(int N) { // expected-note 2{{declared here}} auto l1 = []() -> T {}; // expected-error{{incomplete result type 'Incomplete' in lambda expression}} - // expected-note@-1{{while substituting into a lambda expression here}} typedef int vla[N]; // expected-warning 2{{variable length arrays in C++ are a Clang extension}} \ expected-note 2{{function parameter 'N' with unknown value cannot be used in a constant expression}} auto l2 = []() -> vla {}; // expected-error{{function cannot return array type 'vla' (aka 'int[N]')}} } void test_result_type_call() { - test_result_type_tpl(10); // expected-note 2{{requested here}} + test_result_type_tpl(10); // expected-note {{requested here}} } diff --git a/clang/test/CXX/temp/temp.deduct/p9.cpp b/clang/test/CXX/temp/temp.deduct/p9.cpp index 23bcd2a1892e..7b661c275211 100644 --- a/clang/test/CXX/temp/temp.deduct/p9.cpp +++ b/clang/test/CXX/temp/temp.deduct/p9.cpp @@ -28,18 +28,14 @@ template auto h(T) -> decltype([x = T::invalid]() { }); void h(...); void test_h() { - h(0); // expected-error@-3 {{type 'int' cannot be used prior to '::'}} - // expected-note@-1 {{while substituting deduced template arguments}} - // expected-note@-5 {{while substituting into a lambda expression here}} + h(0); } template auto i(T) -> decltype([]() -> typename T::invalid { }); void i(...); void test_i() { - i(0); // expected-error@-3 {{type 'int' cannot be used prior to '::'}} - // expected-note@-1 {{while substituting deduced template arguments}} - // expected-note@-5 {{while substituting into a lambda expression here}} + i(0); } diff --git a/clang/test/SemaCXX/cxx1y-init-captures.cpp b/clang/test/SemaCXX/cxx1y-init-captures.cpp index 4cb53cc8bfc1..5340c6c7d0bf 100644 --- a/clang/test/SemaCXX/cxx1y-init-captures.cpp +++ b/clang/test/SemaCXX/cxx1y-init-captures.cpp @@ -21,17 +21,17 @@ namespace variadic_expansion { return a; }() ...); }; - auto N2 = [x = y, //expected-note3{{begins here}} expected-note 6 {{default capture by}} expected-note 2 {{substituting into a lambda}} + auto N2 = [x = y, //expected-note3{{begins here}} expected-note 6 {{default capture by}} &z = y, n = f(t...), - o = f([&a(t)](T& ... t)->decltype(auto) { return a; }(t...)...)](T& ... s) { // expected-note 6 {{capture 't' by}} - fv([&a(t)]()->decltype(auto) { //expected-error 3{{captured}} expected-note 2{{substituting into a lambda}} + o = f([&a(t)](T& ... t)->decltype(auto) { return a; }(t...)...)](T& ... s) { // expected-note 6 {{capture 't' by}} expected-note {{substituting into a lambda}} + fv([&a(t)]()->decltype(auto) { //expected-error 3{{captured}} return a; }() ...); }; } - void h(int i, char c) { g(i, c); } //expected-note 2{{in instantiation}} + void h(int i, char c) { g(i, c); } // expected-note {{requested here}} } namespace odr_use_within_init_capture { diff --git a/clang/test/SemaCXX/cxx1z-lambda-star-this.cpp b/clang/test/SemaCXX/cxx1z-lambda-star-this.cpp index 0cee41ff5ed3..95bc32b603dd 100644 --- a/clang/test/SemaCXX/cxx1z-lambda-star-this.cpp +++ b/clang/test/SemaCXX/cxx1z-lambda-star-this.cpp @@ -46,7 +46,7 @@ public: template void foo() { (void)[this] { return x; }; - (void)[*this] { return x; }; //expected-error2{{call to deleted}} expected-note {{while substituting into a lambda}} + (void)[*this] { return x; }; //expected-error2{{call to deleted}} } B() = default; @@ -63,7 +63,7 @@ class B { public: template auto foo() { - const auto &L = [*this](auto a) mutable { //expected-error{{call to deleted}} expected-note {{while substituting into a lambda}} + const auto &L = [*this](auto a) mutable { //expected-error{{call to deleted}} d += a; return [this](auto b) { return d += b; }; }; diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp index 1797eef320b8..41cf5a46c389 100644 --- a/clang/test/SemaCXX/lambda-expressions.cpp +++ b/clang/test/SemaCXX/lambda-expressions.cpp @@ -393,7 +393,7 @@ struct S { namespace PR18473 { template void f() { T t(0); - (void) [=]{ int n = t; }; // expected-error {{deleted}} expected-note {{while substituting into a lambda}} + (void) [=]{ int n = t; }; // expected-error {{deleted}} } template void f(); @@ -476,7 +476,7 @@ namespace error_in_transform_prototype { void f(T t) { // expected-error@+2 {{type 'int' cannot be used prior to '::' because it has no members}} // expected-error@+1 {{no member named 'ns' in 'error_in_transform_prototype::S'}} - auto x = [](typename T::ns::type &k) {}; // expected-note 2 {{while substituting into a lambda}} + auto x = [](typename T::ns::type &k) {}; } class S {}; void foo() { diff --git a/clang/test/SemaCXX/lambda-pack-expansion.cpp b/clang/test/SemaCXX/lambda-pack-expansion.cpp index 936e7c6b0e5c..e3e968e2704e 100644 --- a/clang/test/SemaCXX/lambda-pack-expansion.cpp +++ b/clang/test/SemaCXX/lambda-pack-expansion.cpp @@ -8,7 +8,6 @@ struct X { void take_by_copy(auto &...args) { [...args = args] {}(); // expected-error {{call to deleted constructor}} - // expected-note@-1 {{substituting into a lambda}} } void take_by_ref(auto &...args) { diff --git a/clang/test/SemaCXX/vartemplate-lambda.cpp b/clang/test/SemaCXX/vartemplate-lambda.cpp index d2b53b53dcd4..8b232abe976b 100644 --- a/clang/test/SemaCXX/vartemplate-lambda.cpp +++ b/clang/test/SemaCXX/vartemplate-lambda.cpp @@ -8,7 +8,6 @@ template auto v1 = [](int a = T()) { return a; }(); // expected-error@-1{{cannot initialize a parameter of type 'int' with an rvalue of type 'int *'}} // expected-note@-2{{in instantiation of default function argument expression for 'operator()' required here}} // expected-note@-3{{passing argument to parameter 'a' here}} -// expected-note@-4{{substituting into a lambda}} struct S { template diff --git a/clang/test/SemaCXX/warn-unused-lambda-capture.cpp b/clang/test/SemaCXX/warn-unused-lambda-capture.cpp index 4214a3c6ce09..73459496e4cd 100644 --- a/clang/test/SemaCXX/warn-unused-lambda-capture.cpp +++ b/clang/test/SemaCXX/warn-unused-lambda-capture.cpp @@ -145,38 +145,38 @@ void test_templated() { auto explicit_by_value_used_generic = [i](auto c) { return i + 1; }; auto explicit_by_value_used_void = [i] { (void)i; }; - auto explicit_by_value_unused = [i] {}; // expected-warning{{lambda capture 'i' is not used}} expected-note {{substituting into a lambda}} - auto explicit_by_value_unused_sizeof = [i] { return sizeof(i); }; // expected-warning{{lambda capture 'i' is not required to be captured for this use}} expected-note {{substituting into a lambda}} - auto explicit_by_value_unused_decltype = [i] { decltype(i) j = 0; }; // expected-warning{{lambda capture 'i' is not used}} expected-note {{substituting into a lambda}} - auto explicit_by_value_unused_const = [k] { return k + 1; }; // expected-warning{{lambda capture 'k' is not required to be captured for this use}} expected-note {{substituting into a lambda}} - auto explicit_by_value_unused_const_generic = [k](auto c) { return k + 1; }; // expected-warning{{lambda capture 'k' is not required to be captured for this use}} expected-note {{substituting into a lambda}} + auto explicit_by_value_unused = [i] {}; // expected-warning{{lambda capture 'i' is not used}} + auto explicit_by_value_unused_sizeof = [i] { return sizeof(i); }; // expected-warning{{lambda capture 'i' is not required to be captured for this use}} + auto explicit_by_value_unused_decltype = [i] { decltype(i) j = 0; }; // expected-warning{{lambda capture 'i' is not used}} + auto explicit_by_value_unused_const = [k] { return k + 1; }; // expected-warning{{lambda capture 'k' is not required to be captured for this use}} + auto explicit_by_value_unused_const_generic = [k](auto c) { return k + 1; }; // expected-warning{{lambda capture 'k' is not required to be captured for this use}} auto explicit_by_reference_used = [&i] { i++; }; - auto explicit_by_reference_unused = [&i] {}; // expected-warning{{lambda capture 'i' is not used}} expected-note {{substituting into a lambda}} + auto explicit_by_reference_unused = [&i] {}; // expected-warning{{lambda capture 'i' is not used}} auto explicit_initialized_reference_used = [&j = i] { return j + 1; }; - auto explicit_initialized_reference_unused = [&j = i]{}; // expected-warning{{lambda capture 'j' is not used}} expected-note {{substituting into a lambda}} + auto explicit_initialized_reference_unused = [&j = i]{}; // expected-warning{{lambda capture 'j' is not used}} auto explicit_initialized_value_used = [j = 1] { return j + 1; }; - auto explicit_initialized_value_unused = [j = 1] {}; // expected-warning{{lambda capture 'j' is not used}} expected-note {{substituting into a lambda}} + auto explicit_initialized_value_unused = [j = 1] {}; // expected-warning{{lambda capture 'j' is not used}} auto explicit_initialized_value_non_trivial_constructor = [j = NonTrivialConstructor()]{}; auto explicit_initialized_value_non_trivial_destructor = [j = NonTrivialDestructor()]{}; - auto explicit_initialized_value_trivial_init = [j = Trivial()]{}; // expected-warning{{lambda capture 'j' is not used}} expected-note {{substituting into a lambda}} + auto explicit_initialized_value_trivial_init = [j = Trivial()]{}; // expected-warning{{lambda capture 'j' is not used}} auto explicit_initialized_value_non_trivial_init = [j = Trivial(42)]{}; auto explicit_initialized_value_with_side_effect = [j = side_effect()]{}; auto explicit_initialized_value_generic_used = [i = 1](auto c) mutable { i++; }; - auto explicit_initialized_value_generic_unused = [i = 1](auto c) mutable {}; // expected-warning{{lambda capture 'i' is not used}} expected-note {{substituting into a lambda}} + auto explicit_initialized_value_generic_unused = [i = 1](auto c) mutable {}; // expected-warning{{lambda capture 'i' is not used}} auto nested = [&i] { // expected-note {{substituting into a lambda}} auto explicit_by_value_used = [i] { return i + 1; }; - auto explicit_by_value_unused = [i] {}; // expected-warning{{lambda capture 'i' is not used}} expected-note {{substituting into a lambda}} + auto explicit_by_value_unused = [i] {}; // expected-warning{{lambda capture 'i' is not used}} }; Trivial trivial; - auto explicit_by_value_trivial = [trivial] {}; // expected-warning{{lambda capture 'trivial' is not used}} expected-note {{substituting into a lambda}} + auto explicit_by_value_trivial = [trivial] {}; // expected-warning{{lambda capture 'trivial' is not used}} NonTrivialConstructor cons; - auto explicit_by_value_non_trivial_constructor = [cons] {}; // expected-warning{{lambda capture 'cons' is not used}} expected-note {{substituting into a lambda}} + auto explicit_by_value_non_trivial_constructor = [cons] {}; // expected-warning{{lambda capture 'cons' is not used}} NonTrivialCopyConstructor copy_cons; auto explicit_by_value_non_trivial_copy_constructor = [copy_cons] {}; @@ -189,7 +189,7 @@ void test_templated() { } void test_use_template() { - test_templated(); // expected-note 13{{in instantiation of function template specialization 'test_templated' requested here}} + test_templated(); // expected-note 3 {{requested here}} } namespace pr35555 { diff --git a/clang/test/SemaTemplate/instantiate-local-class.cpp b/clang/test/SemaTemplate/instantiate-local-class.cpp index f9553e334c70..47591045fd26 100644 --- a/clang/test/SemaTemplate/instantiate-local-class.cpp +++ b/clang/test/SemaTemplate/instantiate-local-class.cpp @@ -473,8 +473,7 @@ namespace rdar23721638 { template void bar() { auto lambda = [](T a = "") {}; // expected-error {{conversion function from 'const char[1]' to 'rdar23721638::A' invokes a deleted function}} \ // expected-note {{in instantiation of default function argument expression for 'operator()' required here}} \ - // expected-note {{passing argument to parameter 'a' here}} \ - // expected-note {{while substituting into a lambda}} + // expected-note {{passing argument to parameter 'a' here}} lambda(); } template void bar(); // expected-note {{in instantiation}} @@ -497,7 +496,6 @@ namespace PR45000 { // expected-error@-1 {{cannot initialize a parameter of type 'int' with an rvalue of type 'std::nullptr_t'}} // expected-note@-2 {{in instantiation of default function argument expression for 'operator()' required here}} // expected-note@-3 {{passing argument to parameter 'x' here}} - // expected-note@-4 {{while substituting into a lambda}} void g() { f(); } // expected-note@-1 {{in instantiation of default function argument expression for 'f' required here}} -- GitLab From d187dfe51534b8e15834346ea1898977b7bedace Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 5 Jan 2024 12:39:32 +0000 Subject: [PATCH 102/728] [AArch64] Add some tests for addLikeOr with csinc. NFC --- llvm/test/CodeGen/AArch64/arm64-csel.ll | 64 +++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/llvm/test/CodeGen/AArch64/arm64-csel.ll b/llvm/test/CodeGen/AArch64/arm64-csel.ll index 35051d0c6017..bda0c6509101 100644 --- a/llvm/test/CodeGen/AArch64/arm64-csel.ll +++ b/llvm/test/CodeGen/AArch64/arm64-csel.ll @@ -420,3 +420,67 @@ entry: ret i64 %add } +define i32 @or(i32 %num, i32 %x) { +; CHECK-LABEL: or: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: cmp w1, #0 +; CHECK-NEXT: and w8, w0, #0xff00 +; CHECK-NEXT: cset w9, ne +; CHECK-NEXT: orr w0, w8, w9 +; CHECK-NEXT: ret +entry: + %and = and i32 %num, 65280 + %tobool.not = icmp ne i32 %x, 0 + %cond = zext i1 %tobool.not to i32 + %or = or disjoint i32 %and, %cond + ret i32 %or +} + +define i64 @or64(i64 %num, i64 %x) { +; CHECK-LABEL: or64: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: cmp x1, #0 +; CHECK-NEXT: and x8, x0, #0xff00 +; CHECK-NEXT: cset w9, ne +; CHECK-NEXT: orr x0, x8, x9 +; CHECK-NEXT: ret +entry: + %and = and i64 %num, 65280 + %tobool.not = icmp ne i64 %x, 0 + %conv = zext i1 %tobool.not to i64 + %or = or disjoint i64 %and, %conv + ret i64 %or +} + +define i32 @selor32(i32 %num, i32 %x) { +; CHECK-LABEL: selor32: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and w8, w0, #0xff00 +; CHECK-NEXT: cmp w1, #0 +; CHECK-NEXT: orr w9, w8, #0x1 +; CHECK-NEXT: csel w0, w9, w8, ne +; CHECK-NEXT: ret +entry: + %and = and i32 %num, 65280 + %tobool.not = icmp ne i32 %x, 0 + %or = or disjoint i32 %and, 1 + %sel = select i1 %tobool.not, i32 %or, i32 %and + ret i32 %sel +} + + +define i64 @selor64(i64 %num, i64 %x) { +; CHECK-LABEL: selor64: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: and x8, x0, #0xff00 +; CHECK-NEXT: cmp x1, #0 +; CHECK-NEXT: orr x9, x8, #0x1 +; CHECK-NEXT: csel x0, x9, x8, ne +; CHECK-NEXT: ret +entry: + %and = and i64 %num, 65280 + %tobool.not = icmp ne i64 %x, 0 + %or = or disjoint i64 %and, 1 + %sel = select i1 %tobool.not, i64 %or, i64 %and + ret i64 %sel +} -- GitLab From 86127305d4602801ba7ffdc74377ed67b18819ac Mon Sep 17 00:00:00 2001 From: Qizhi Hu <836744285@qq.com> Date: Fri, 5 Jan 2024 20:53:04 +0800 Subject: [PATCH 103/728] [clang][ASTImporter][StructuralEquivalence] improve StructuralEquivalence on recordType (#76226) Types comparison in `StructuralEquivalence` ignores its `DeclContext` when they are generated by template specialization implicitly and this will produce incorrect result. Add comparison of `DeclContext` of ClassTemplateSpecializationDecl to improve result. fix [issue](https://github.com/llvm/llvm-project/issues/65913) Co-authored-by: huqizhi <836744285@qq.com> --- clang/lib/AST/ASTStructuralEquivalence.cpp | 13 ++++++++--- .../AST/StructuralEquivalenceTest.cpp | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp index 1f492b051e03..a9e0d1698a91 100644 --- a/clang/lib/AST/ASTStructuralEquivalence.cpp +++ b/clang/lib/AST/ASTStructuralEquivalence.cpp @@ -1463,8 +1463,9 @@ IsStructurallyEquivalentLambdas(StructuralEquivalenceContext &Context, } /// Determine if context of a class is equivalent. -static bool IsRecordContextStructurallyEquivalent(RecordDecl *D1, - RecordDecl *D2) { +static bool +IsRecordContextStructurallyEquivalent(StructuralEquivalenceContext &Context, + RecordDecl *D1, RecordDecl *D2) { // The context should be completely equal, including anonymous and inline // namespaces. // We compare objects as part of full translation units, not subtrees of @@ -1491,6 +1492,12 @@ static bool IsRecordContextStructurallyEquivalent(RecordDecl *D1, return false; } + if (auto *D1Spec = dyn_cast(DC1)) { + auto *D2Spec = dyn_cast(DC2); + if (!IsStructurallyEquivalent(Context, D1Spec, D2Spec)) + return false; + } + DC1 = DC1->getParent()->getNonTransparentContext(); DC2 = DC2->getParent()->getNonTransparentContext(); } @@ -1544,7 +1551,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context, // If the records occur in different context (namespace), these should be // different. This is specially important if the definition of one or both // records is missing. - if (!IsRecordContextStructurallyEquivalent(D1, D2)) + if (!IsRecordContextStructurallyEquivalent(Context, D1, D2)) return false; // If both declarations are class template specializations, we know diff --git a/clang/unittests/AST/StructuralEquivalenceTest.cpp b/clang/unittests/AST/StructuralEquivalenceTest.cpp index 44d950cfe758..22c7b82460f0 100644 --- a/clang/unittests/AST/StructuralEquivalenceTest.cpp +++ b/clang/unittests/AST/StructuralEquivalenceTest.cpp @@ -1024,6 +1024,29 @@ TEST_F(StructuralEquivalenceRecordContextTest, TransparentContextInNamespace) { EXPECT_TRUE(testStructuralMatch(Decls)); } +TEST_F(StructuralEquivalenceRecordContextTest, + ClassTemplateSpecializationContext) { + std::string Code = + R"( + template struct O { + struct M {}; + }; + )"; + auto t = makeDecls(Code + R"( + typedef O::M MT1; + MT1 A; + )", + Code + R"( + namespace { + struct I {}; + } // namespace + typedef O::M MT2; + MT2 A; + )", + Lang_CXX11, varDecl(hasName("A"))); + EXPECT_FALSE(testStructuralMatch(t)); +} + TEST_F(StructuralEquivalenceTest, NamespaceOfRecordMember) { auto Decls = makeNamedDecls( R"( -- GitLab From 70e67253d60a984bd83844690791a023448e3623 Mon Sep 17 00:00:00 2001 From: Abhina Sree <69635948+abhina-sree@users.noreply.github.com> Date: Fri, 5 Jan 2024 07:57:07 -0500 Subject: [PATCH 104/728] [System][z/OS] Fix per-thread timing error on z/OS (#76947) This patch calls ProcessCPUUsage() for z/OS because we do not support CLOCK_THREAD_CPUTIME_ID. This copies the change made to google benchmark here https://github.com/google/benchmark/commit/e3824e7503187993b287ac8c8144a35bf5ccfd44 --- third-party/benchmark/src/timers.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/third-party/benchmark/src/timers.cc b/third-party/benchmark/src/timers.cc index 21d3db20da9e..ed35c01f5540 100644 --- a/third-party/benchmark/src/timers.cc +++ b/third-party/benchmark/src/timers.cc @@ -168,6 +168,9 @@ double ThreadCPUUsage() { // RTEMS doesn't support CLOCK_THREAD_CPUTIME_ID. See // https://github.com/RTEMS/rtems/blob/master/cpukit/posix/src/clockgettime.c return ProcessCPUUsage(); +#elif defined(BENCHMARK_OS_ZOS) + // z/OS doesn't support CLOCK_THREAD_CPUTIME_ID. + return ProcessCPUUsage(); #elif defined(BENCHMARK_OS_SOLARIS) struct rusage ru; if (getrusage(RUSAGE_LWP, &ru) == 0) return MakeTime(ru); -- GitLab From f7f7574afe4cfc11ebe5d8cb811d5cd28dc862f6 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Fri, 5 Jan 2024 21:03:24 +0800 Subject: [PATCH 105/728] [InstCombine] Canonicalize `switch(C-X)` to `switch(X)` (#77051) This patch canonicalizes `switch(C-X)` to `switch(X)`. Compile-time impact: http://llvm-compile-time-tracker.com/compare.php?from=7954c57124b495fbdc73674d71f2e366e4afe522&to=31a9adff1e633f0f3c423fb8487fc15d17e171f2&stat=instructions:u |stage1-O3|stage1-ReleaseThinLTO|stage1-ReleaseLTO-g|stage1-O0-g|stage2-O3|stage2-O0-g|stage2-clang| |--|--|--|--|--|--|--| |-0.01%|+0.02%|+0.02%|+0.05%|-0.07%|-0.02%|-0.02%| --- .../InstCombine/InstructionCombining.cpp | 12 +++ .../test/Transforms/InstCombine/switch-sub.ll | 89 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 llvm/test/Transforms/InstCombine/switch-sub.ll diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index f3181dc14792..b45c6784b85b 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -3208,6 +3208,18 @@ Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) { return replaceOperand(SI, 0, Op0); } + ConstantInt *SubLHS; + if (match(Cond, m_Sub(m_ConstantInt(SubLHS), m_Value(Op0)))) { + // Change 'switch (1-X) case 1:' into 'switch (X) case 0'. + for (auto Case : SI.cases()) { + Constant *NewCase = ConstantExpr::getSub(SubLHS, Case.getCaseValue()); + assert(isa(NewCase) && + "Result of expression should be constant"); + Case.setValue(cast(NewCase)); + } + return replaceOperand(SI, 0, Op0); + } + KnownBits Known = computeKnownBits(Cond, 0, &SI); unsigned LeadingKnownZeros = Known.countMinLeadingZeros(); unsigned LeadingKnownOnes = Known.countMinLeadingOnes(); diff --git a/llvm/test/Transforms/InstCombine/switch-sub.ll b/llvm/test/Transforms/InstCombine/switch-sub.ll new file mode 100644 index 000000000000..3fedfddeb7c0 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/switch-sub.ll @@ -0,0 +1,89 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +define i1 @test_switch_with_neg(i32 %a) { +; CHECK-LABEL: define i1 @test_switch_with_neg( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: switch i32 [[A]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 -37, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 -38, label [[SW_BB]] +; CHECK-NEXT: i32 -39, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %a.neg = sub i32 0, %a + switch i32 %a.neg, label %sw.default [ + i32 37, label %sw.bb + i32 38, label %sw.bb + i32 39, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +define i1 @test_switch_with_sub(i32 %a) { +; CHECK-LABEL: define i1 @test_switch_with_sub( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: switch i32 [[A]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 -1, label [[SW_BB]] +; CHECK-NEXT: i32 -2, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %a.neg = sub i32 37, %a + switch i32 %a.neg, label %sw.default [ + i32 37, label %sw.bb + i32 38, label %sw.bb + i32 39, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +; Negative tests + +define i1 @test_switch_with_sub_nonconst(i32 %a, i32 %b) { +; CHECK-LABEL: define i1 @test_switch_with_sub_nonconst( +; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_NEG:%.*]] = sub i32 [[B]], [[A]] +; CHECK-NEXT: switch i32 [[A_NEG]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 37, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 38, label [[SW_BB]] +; CHECK-NEXT: i32 39, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %a.neg = sub i32 %b, %a + switch i32 %a.neg, label %sw.default [ + i32 37, label %sw.bb + i32 38, label %sw.bb + i32 39, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} -- GitLab From 0504e932880e0c4c333baeba9b8d674c5b9ae316 Mon Sep 17 00:00:00 2001 From: Nishant Mittal Date: Fri, 5 Jan 2024 18:53:23 +0530 Subject: [PATCH 106/728] [libc][math] Implement nan(f|l) functions (#76690) Specification: https://en.cppreference.com/w/c/numeric/math/nan --- libc/config/darwin/arm/entrypoints.txt | 3 + libc/config/darwin/x86_64/entrypoints.txt | 3 + libc/config/gpu/entrypoints.txt | 2 + libc/config/linux/aarch64/entrypoints.txt | 3 + libc/config/linux/riscv/entrypoints.txt | 3 + libc/config/linux/x86_64/entrypoints.txt | 3 + libc/config/windows/entrypoints.txt | 3 + libc/docs/math/index.rst | 6 +- libc/spec/stdc.td | 4 ++ libc/src/__support/str_to_float.h | 65 ++++++++++++----- libc/src/math/CMakeLists.txt | 4 ++ libc/src/math/generic/CMakeLists.txt | 39 ++++++++++ libc/src/math/generic/nan.cpp | 23 ++++++ libc/src/math/generic/nanf.cpp | 23 ++++++ libc/src/math/generic/nanl.cpp | 23 ++++++ libc/src/math/nan.h | 18 +++++ libc/src/math/nanf.h | 18 +++++ libc/src/math/nanl.h | 18 +++++ libc/test/src/math/smoke/CMakeLists.txt | 39 ++++++++++ libc/test/src/math/smoke/nan_test.cpp | 47 ++++++++++++ libc/test/src/math/smoke/nanf_test.cpp | 46 ++++++++++++ libc/test/src/math/smoke/nanl_test.cpp | 72 +++++++++++++++++++ .../llvm-project-overlay/libc/BUILD.bazel | 6 ++ 23 files changed, 451 insertions(+), 20 deletions(-) create mode 100644 libc/src/math/generic/nan.cpp create mode 100644 libc/src/math/generic/nanf.cpp create mode 100644 libc/src/math/generic/nanl.cpp create mode 100644 libc/src/math/nan.h create mode 100644 libc/src/math/nanf.h create mode 100644 libc/src/math/nanl.h create mode 100644 libc/test/src/math/smoke/nan_test.cpp create mode 100644 libc/test/src/math/smoke/nanf_test.cpp create mode 100644 libc/test/src/math/smoke/nanl_test.cpp diff --git a/libc/config/darwin/arm/entrypoints.txt b/libc/config/darwin/arm/entrypoints.txt index da4a2345f389..02a092566069 100644 --- a/libc/config/darwin/arm/entrypoints.txt +++ b/libc/config/darwin/arm/entrypoints.txt @@ -193,6 +193,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.modf libc.src.math.modff libc.src.math.modfl + libc.src.math.nan + libc.src.math.nanf + libc.src.math.nanl libc.src.math.nearbyint libc.src.math.nearbyintf libc.src.math.nearbyintl diff --git a/libc/config/darwin/x86_64/entrypoints.txt b/libc/config/darwin/x86_64/entrypoints.txt index e29e75ec92da..91493cb77b1d 100644 --- a/libc/config/darwin/x86_64/entrypoints.txt +++ b/libc/config/darwin/x86_64/entrypoints.txt @@ -172,6 +172,9 @@ set(TARGET_LIBM_ENTRYPOINTS #libc.src.math.modf #libc.src.math.modff #libc.src.math.modfl + #libc.src.math.nan + #libc.src.math.nanf + #libc.src.math.nanl #libc.src.math.nearbyint #libc.src.math.nearbyintf #libc.src.math.nearbyintl diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt index ba86e31ee0ad..b333c6be1446 100644 --- a/libc/config/gpu/entrypoints.txt +++ b/libc/config/gpu/entrypoints.txt @@ -215,6 +215,8 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.lroundf libc.src.math.modf libc.src.math.modff + libc.src.math.nan + libc.src.math.nanf libc.src.math.nearbyint libc.src.math.nearbyintf libc.src.math.nextafter diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 77c9a50b8b7e..ce3f5eb40e38 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -313,6 +313,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.modf libc.src.math.modff libc.src.math.modfl + libc.src.math.nan + libc.src.math.nanf + libc.src.math.nanl libc.src.math.nearbyint libc.src.math.nearbyintf libc.src.math.nearbyintl diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index e389936ffca1..ec2a16f5cf47 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -322,6 +322,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.modf libc.src.math.modff libc.src.math.modfl + libc.src.math.nan + libc.src.math.nanf + libc.src.math.nanl libc.src.math.nearbyint libc.src.math.nearbyintf libc.src.math.nearbyintl diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 3adcd57d0c08..30900de365bf 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -323,6 +323,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.modf libc.src.math.modff libc.src.math.modfl + libc.src.math.nan + libc.src.math.nanf + libc.src.math.nanl libc.src.math.nearbyint libc.src.math.nearbyintf libc.src.math.nearbyintl diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt index 4c0a6ec37fe4..5c3a2e287b95 100644 --- a/libc/config/windows/entrypoints.txt +++ b/libc/config/windows/entrypoints.txt @@ -192,6 +192,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.modf libc.src.math.modff libc.src.math.modfl + libc.src.math.nan + libc.src.math.nanf + libc.src.math.nanl libc.src.math.nearbyint libc.src.math.nearbyintf libc.src.math.nearbyintl diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst index eaa7a40a29ae..3668524af03c 100644 --- a/libc/docs/math/index.rst +++ b/libc/docs/math/index.rst @@ -212,11 +212,11 @@ Basic Operations +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | modfl | |check| | |check| | | |check| | |check| | | | |check| | | | | | +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| nan | | | | | | | | | | | | | +| nan | |check| | |check| | | |check| | |check| | | | |check| | | | | | +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| nanf | | | | | | | | | | | | | +| nanf | |check| | |check| | | |check| | |check| | | | |check| | | | | | +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| nanl | | | | | | | | | | | | | +| nanl | |check| | |check| | | |check| | |check| | | | |check| | | | | | +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ | nearbyint | |check| | |check| | | |check| | |check| | | | |check| | | | | | +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 58c95b856535..78095eb23f07 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -515,6 +515,10 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"scalbn", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"scalbnf", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"scalbnl", RetValSpec, [ArgSpec, ArgSpec]>, + + FunctionSpec<"nanf", RetValSpec, [ArgSpec]>, + FunctionSpec<"nan", RetValSpec, [ArgSpec]>, + FunctionSpec<"nanl", RetValSpec, [ArgSpec]>, ] >; diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h index be4b55645417..f2d4ec7fb11f 100644 --- a/libc/src/__support/str_to_float.h +++ b/libc/src/__support/str_to_float.h @@ -12,6 +12,7 @@ #include "src/__support/CPP/bit.h" #include "src/__support/CPP/limits.h" #include "src/__support/CPP/optional.h" +#include "src/__support/CPP/string_view.h" #include "src/__support/FPUtil/FEnvImpl.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/dyadic_float.h" @@ -1044,6 +1045,27 @@ hexadecimal_string_to_float(const char *__restrict src, return output; } +LIBC_INLINE uint64_t +nan_mantissa_from_ncharseq(const cpp::string_view ncharseq) { + uint64_t nan_mantissa = 0; + + if (ncharseq.data() != nullptr && isdigit(ncharseq[0])) { + // This is to prevent errors when StorageType is larger than 64 + // bits, since strtointeger only supports up to 64 bits. This is + // actually more than is required by the specification, which says + // for the input type "NAN(n-char-sequence)" that "the meaning of + // the n-char sequence is implementation-defined." + auto strtoint_result = strtointeger(ncharseq.data(), 0); + if (!strtoint_result.has_error()) + nan_mantissa = strtoint_result.value; + + if (strtoint_result.parsed_len != static_cast(ncharseq.size())) + nan_mantissa = 0; + } + + return nan_mantissa; +} + // Takes a pointer to a string and a pointer to a string pointer. This function // is used as the backend for all of the string to float functions. template @@ -1136,31 +1158,18 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { ++index; if (src[index] == ')') { ++index; - if (isdigit(src[left_paren + 1])) { - // This is to prevent errors when StorageType is larger than 64 - // bits, since strtointeger only supports up to 64 bits. This is - // actually more than is required by the specification, which says - // for the input type "NAN(n-char-sequence)" that "the meaning of - // the n-char sequence is implementation-defined." - - auto strtoint_result = - strtointeger(src + (left_paren + 1), 0); - if (strtoint_result.has_error()) { - error = strtoint_result.error; - } - nan_mantissa = static_cast(strtoint_result.value); - if (src[left_paren + 1 + strtoint_result.parsed_len] != ')') - nan_mantissa = 0; - } + auto nan_mantissa_result = nan_mantissa_from_ncharseq( + cpp::string_view(src + (left_paren + 1), index - left_paren - 2)); + nan_mantissa = static_cast(nan_mantissa_result); } else { index = left_paren; } } + if (result.get_sign()) { result = FPBits(result.build_quiet_nan(nan_mantissa)); result.set_sign(true); } else { - result.set_sign(false); result = FPBits(result.build_quiet_nan(nan_mantissa)); } } @@ -1195,6 +1204,28 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { return {T(result), index, error}; } +template LIBC_INLINE StrToNumResult strtonan(const char *arg) { + using FPBits = typename fputil::FPBits; + using StorageType = typename FPBits::StorageType; + + FPBits result; + int error = 0; + StorageType nan_mantissa = 0; + + ptrdiff_t index = 0; + while (isalnum(arg[index]) || arg[index] == '_') + ++index; + + if (arg[index] == '\0') { + auto nan_mantissa_result = + nan_mantissa_from_ncharseq(cpp::string_view(arg, index)); + nan_mantissa = static_cast(nan_mantissa_result); + } + + result = FPBits(result.build_quiet_nan(nan_mantissa)); + return {T(result), 0, error}; +} + } // namespace internal } // namespace LIBC_NAMESPACE diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt index ffabc27bc00a..e2b1026fcad7 100644 --- a/libc/src/math/CMakeLists.txt +++ b/libc/src/math/CMakeLists.txt @@ -179,6 +179,10 @@ add_math_entrypoint_object(modf) add_math_entrypoint_object(modff) add_math_entrypoint_object(modfl) +add_math_entrypoint_object(nan) +add_math_entrypoint_object(nanf) +add_math_entrypoint_object(nanl) + add_math_entrypoint_object(nearbyint) add_math_entrypoint_object(nearbyintf) add_math_entrypoint_object(nearbyintl) diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index 214d57842d93..eeb09652961f 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -1437,6 +1437,45 @@ add_entrypoint_object( -O3 ) +add_entrypoint_object( + nan + SRCS + nan.cpp + HDRS + ../nan.h + DEPENDS + libc.src.__support.str_to_float + libc.src.errno.errno + COMPILE_OPTIONS + -O3 +) + +add_entrypoint_object( + nanf + SRCS + nanf.cpp + HDRS + ../nanf.h + DEPENDS + libc.src.__support.str_to_float + libc.src.errno.errno + COMPILE_OPTIONS + -O3 +) + +add_entrypoint_object( + nanl + SRCS + nanl.cpp + HDRS + ../nanl.h + DEPENDS + libc.src.__support.str_to_float + libc.src.errno.errno + COMPILE_OPTIONS + -O3 +) + add_entrypoint_object( nextafter SRCS diff --git a/libc/src/math/generic/nan.cpp b/libc/src/math/generic/nan.cpp new file mode 100644 index 000000000000..c0d7fbf4ba48 --- /dev/null +++ b/libc/src/math/generic/nan.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of nan function ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nan.h" +#include "src/__support/common.h" +#include "src/__support/str_to_float.h" +#include "src/errno/libc_errno.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(double, nan, (const char *arg)) { + auto result = internal::strtonan(arg); + if (result.has_error()) + libc_errno = result.error; + return result.value; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/nanf.cpp b/libc/src/math/generic/nanf.cpp new file mode 100644 index 000000000000..2751a81ac461 --- /dev/null +++ b/libc/src/math/generic/nanf.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of nanf function -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nanf.h" +#include "src/__support/common.h" +#include "src/__support/str_to_float.h" +#include "src/errno/libc_errno.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(float, nanf, (const char *arg)) { + auto result = internal::strtonan(arg); + if (result.has_error()) + libc_errno = result.error; + return result.value; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/generic/nanl.cpp b/libc/src/math/generic/nanl.cpp new file mode 100644 index 000000000000..76dcb56b47b2 --- /dev/null +++ b/libc/src/math/generic/nanl.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of nanl function -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nanl.h" +#include "src/__support/common.h" +#include "src/__support/str_to_float.h" +#include "src/errno/libc_errno.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(long double, nanl, (const char *arg)) { + auto result = internal::strtonan(arg); + if (result.has_error()) + libc_errno = result.error; + return result.value; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/math/nan.h b/libc/src/math/nan.h new file mode 100644 index 000000000000..463940b01a27 --- /dev/null +++ b/libc/src/math/nan.h @@ -0,0 +1,18 @@ +//===-- Implementation header for nan ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NAN_H +#define LLVM_LIBC_SRC_MATH_NAN_H + +namespace LIBC_NAMESPACE { + +double nan(const char *arg); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NAN_H diff --git a/libc/src/math/nanf.h b/libc/src/math/nanf.h new file mode 100644 index 000000000000..f05d60e3a967 --- /dev/null +++ b/libc/src/math/nanf.h @@ -0,0 +1,18 @@ +//===-- Implementation header for nanf --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NANF_H +#define LLVM_LIBC_SRC_MATH_NANF_H + +namespace LIBC_NAMESPACE { + +float nanf(const char *arg); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NANF_H diff --git a/libc/src/math/nanl.h b/libc/src/math/nanl.h new file mode 100644 index 000000000000..d8bbce7cc1bc --- /dev/null +++ b/libc/src/math/nanl.h @@ -0,0 +1,18 @@ +//===-- Implementation header for nanl --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_NANL_H +#define LLVM_LIBC_SRC_MATH_NANL_H + +namespace LIBC_NAMESPACE { + +long double nanl(const char *arg); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_MATH_NANL_H diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt index 6ebd374d04a5..65dc80c2a882 100644 --- a/libc/test/src/math/smoke/CMakeLists.txt +++ b/libc/test/src/math/smoke/CMakeLists.txt @@ -1203,6 +1203,45 @@ add_fp_unittest( libc.src.__support.FPUtil.fp_bits ) +add_fp_unittest( + nanf_test + SUITE + libc-math-smoke-tests + SRCS + nanf_test.cpp + DEPENDS + libc.include.math + libc.include.signal + libc.src.math.nanf + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + nan_test + SUITE + libc-math-smoke-tests + SRCS + nan_test.cpp + DEPENDS + libc.include.math + libc.include.signal + libc.src.math.nan + libc.src.__support.FPUtil.fp_bits +) + +add_fp_unittest( + nanl_test + SUITE + libc-math-smoke-tests + SRCS + nanl_test.cpp + DEPENDS + libc.include.math + libc.include.signal + libc.src.math.nanl + libc.src.__support.FPUtil.fp_bits +) + # FIXME: These tests are currently spurious for NVPTX. if(NOT LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX) add_fp_unittest( diff --git a/libc/test/src/math/smoke/nan_test.cpp b/libc/test/src/math/smoke/nan_test.cpp new file mode 100644 index 000000000000..ae6da143ae41 --- /dev/null +++ b/libc/test/src/math/smoke/nan_test.cpp @@ -0,0 +1,47 @@ +//===-- Unittests for nan -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/FPUtil/FPBits.h" +#include "src/math/nan.h" +#include "test/UnitTest/FPMatcher.h" +#include "test/UnitTest/Test.h" +#include + +class LlvmLibcNanTest : public LIBC_NAMESPACE::testing::Test { +public: + using StorageType = LIBC_NAMESPACE::fputil::FPBits::StorageType; + + void run_test(const char *input_str, StorageType bits) { + double result = LIBC_NAMESPACE::nan(input_str); + auto actual_fp = LIBC_NAMESPACE::fputil::FPBits(result); + auto expected_fp = LIBC_NAMESPACE::fputil::FPBits(bits); + EXPECT_EQ(actual_fp.bits, expected_fp.bits); + }; +}; + +TEST_F(LlvmLibcNanTest, NCharSeq) { + run_test("", 0x7ff8000000000000); + run_test("1234", 0x7ff80000000004d2); + run_test("0x1234", 0x7ff8000000001234); + run_test("1a", 0x7ff8000000000000); + run_test("1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM_", + 0x7ff8000000000000); + run_test("10000000000000000000000000000000000000000000000000", + 0x7ff8000000000000); +} + +TEST_F(LlvmLibcNanTest, RandomString) { + run_test(" 1234", 0x7ff8000000000000); + run_test("-1234", 0x7ff8000000000000); + run_test("asd&f", 0x7ff8000000000000); + run_test("123 ", 0x7ff8000000000000); +} + +TEST_F(LlvmLibcNanTest, InvalidInput) { + EXPECT_DEATH([] { LIBC_NAMESPACE::nan(nullptr); }, WITH_SIGNAL(SIGSEGV)); +} diff --git a/libc/test/src/math/smoke/nanf_test.cpp b/libc/test/src/math/smoke/nanf_test.cpp new file mode 100644 index 000000000000..a602aa9f8ceb --- /dev/null +++ b/libc/test/src/math/smoke/nanf_test.cpp @@ -0,0 +1,46 @@ +//===-- Unittests for nanf ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/FPUtil/FPBits.h" +#include "src/math/nanf.h" +#include "test/UnitTest/FPMatcher.h" +#include "test/UnitTest/Test.h" +#include + +class LlvmLibcNanfTest : public LIBC_NAMESPACE::testing::Test { +public: + using StorageType = LIBC_NAMESPACE::fputil::FPBits::StorageType; + + void run_test(const char *input_str, StorageType bits) { + float result = LIBC_NAMESPACE::nanf(input_str); + auto actual_fp = LIBC_NAMESPACE::fputil::FPBits(result); + auto expected_fp = LIBC_NAMESPACE::fputil::FPBits(bits); + EXPECT_EQ(actual_fp.bits, expected_fp.bits); + }; +}; + +TEST_F(LlvmLibcNanfTest, NCharSeq) { + run_test("", 0x7fc00000); + run_test("1234", 0x7fc004d2); + run_test("0x1234", 0x7fc01234); + run_test("1a", 0x7fc00000); + run_test("1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM_", + 0x7fc00000); + run_test("10000000000000000000000000000000000000000000000000", 0x7fc00000); +} + +TEST_F(LlvmLibcNanfTest, RandomString) { + run_test(" 1234", 0x7fc00000); + run_test("-1234", 0x7fc00000); + run_test("asd&f", 0x7fc00000); + run_test("123 ", 0x7fc00000); +} + +TEST_F(LlvmLibcNanfTest, InvalidInput) { + EXPECT_DEATH([] { LIBC_NAMESPACE::nanf(nullptr); }, WITH_SIGNAL(SIGSEGV)); +} diff --git a/libc/test/src/math/smoke/nanl_test.cpp b/libc/test/src/math/smoke/nanl_test.cpp new file mode 100644 index 000000000000..c0e954016112 --- /dev/null +++ b/libc/test/src/math/smoke/nanl_test.cpp @@ -0,0 +1,72 @@ +//===-- Unittests for nanl ------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/FPUtil/FPBits.h" +#include "src/math/nanl.h" +#include "test/UnitTest/FPMatcher.h" +#include "test/UnitTest/Test.h" +#include + +#if defined(LIBC_LONG_DOUBLE_IS_FLOAT64) +#define SELECT_LONG_DOUBLE(val, _, __) val +#elif defined(LIBC_LONG_DOUBLE_IS_X86_FLOAT80) +#define SELECT_LONG_DOUBLE(_, val, __) val +#else +#define SELECT_LONG_DOUBLE(_, __, val) val +#endif + +class LlvmLibcNanlTest : public LIBC_NAMESPACE::testing::Test { +public: + using StorageType = LIBC_NAMESPACE::fputil::FPBits::StorageType; + + void run_test(const char *input_str, StorageType bits) { + long double result = LIBC_NAMESPACE::nanl(input_str); + auto actual_fp = LIBC_NAMESPACE::fputil::FPBits(result); + auto expected_fp = LIBC_NAMESPACE::fputil::FPBits(bits); + EXPECT_EQ(actual_fp.bits, expected_fp.bits); + }; +}; + +TEST_F(LlvmLibcNanlTest, NCharSeq) { + run_test("", + SELECT_LONG_DOUBLE(0x7ff8000000000000, (UInt128(0x7fffc00000) << 40), + (UInt128(0x7fff800000000000) << 64))); + run_test("1234", SELECT_LONG_DOUBLE( + 0x7ff80000000004d2, + (UInt128(0x7fffc00000) << 40) + UInt128(0x4d2), + (UInt128(0x7fff800000000000) << 64) + UInt128(0x4d2))); + run_test("0x1234", + SELECT_LONG_DOUBLE(0x7ff8000000001234, + (UInt128(0x7fffc00000) << 40) + UInt128(0x1234), + (UInt128(0x7fff800000000000) << 64) + + UInt128(0x1234))); + run_test("1a", + SELECT_LONG_DOUBLE(0x7ff8000000000000, (UInt128(0x7fffc00000) << 40), + (UInt128(0x7fff800000000000) << 64))); + run_test("1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM_", + SELECT_LONG_DOUBLE(0x7ff8000000000000, (UInt128(0x7fffc00000) << 40), + (UInt128(0x7fff800000000000) << 64))); + run_test("10000000000000000000000000000000000000000000000000", + SELECT_LONG_DOUBLE(0x7ff8000000000000, (UInt128(0x7fffc00000) << 40), + (UInt128(0x7fff800000000000) << 64))); +} + +TEST_F(LlvmLibcNanlTest, RandomString) { + StorageType expected = + SELECT_LONG_DOUBLE(0x7ff8000000000000, (UInt128(0x7fffc00000) << 40), + (UInt128(0x7fff800000000000) << 64)); + + run_test(" 1234", expected); + run_test("-1234", expected); + run_test("asd&f", expected); + run_test("123 ", expected); +} + +TEST_F(LlvmLibcNanlTest, InvalidInput) { + EXPECT_DEATH([] { LIBC_NAMESPACE::nanl(nullptr); }, WITH_SIGNAL(SIGSEGV)); +} diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index b5238f7686e5..7d2bb2980255 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1874,6 +1874,12 @@ libc_math_function(name = "llroundf") libc_math_function(name = "llroundl") +libc_math_function(name = "nan") + +libc_math_function(name = "nanf") + +libc_math_function(name = "nanl") + libc_math_function(name = "nearbyint") libc_math_function(name = "nearbyintf") -- GitLab From f8e2e25a09cb0018c9310d124e2788e04a0d36ed Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Fri, 5 Jan 2024 08:24:58 -0500 Subject: [PATCH 107/728] [gn] port 255f95a40377 --- llvm/utils/gn/secondary/libcxx/src/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn index 124d7b193398..5ba8a7cf4b91 100644 --- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn @@ -122,6 +122,7 @@ cxx_sources = [ "condition_variable_destructor.cpp", "error_category.cpp", "exception.cpp", + "fstream.cpp", "functional.cpp", "future.cpp", "hash.cpp", -- GitLab From da148a0805f1e1057b90d282321dea7966744201 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 5 Jan 2024 13:32:00 +0000 Subject: [PATCH 108/728] [AArch64] Add tests showing unnecessary cast promotion. --- .../AArch64/avoid-free-ext-promotion.ll | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll diff --git a/llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll b/llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll new file mode 100644 index 000000000000..35f871e504ca --- /dev/null +++ b/llvm/test/CodeGen/AArch64/avoid-free-ext-promotion.ll @@ -0,0 +1,146 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=arm64-apple-macosx -o - %s | FileCheck %s + +%struct.zot = type { ptr, i32, ptr, ptr, i32, i32, i32, i32, i32, i32 } + +; FIXME: currently the AND is performed on 64 bit unnecessarily. + +; Test cases where an AND is extended from i32 -> i64 which is free. Make +; sure the extends do not get moved to the arguments, which would perform the +; AND on 64 bits unnecessarily. + +define void @avoid_promotion_1_and(ptr nocapture noundef %arg, ptr %p) { +; CHECK-LABEL: avoid_promotion_1_and: +; CHECK: ; %bb.0: ; %bb +; CHECK-NEXT: ldr w8, [x0, #52] +; CHECK-NEXT: mov w9, #10 ; =0xa +; CHECK-NEXT: LBB0_1: ; %bb8 +; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: cmp w8, #3 +; CHECK-NEXT: b.lo LBB0_1 +; CHECK-NEXT: ; %bb.2: ; %bb9 +; CHECK-NEXT: ; in Loop: Header=BB0_1 Depth=1 +; CHECK-NEXT: ldr w10, [x0, #32] +; CHECK-NEXT: ldr w11, [x1, #76] +; CHECK-NEXT: ldr w12, [x1] +; CHECK-NEXT: eor w10, w10, w11 +; CHECK-NEXT: and x10, x10, x12 +; CHECK-NEXT: str w10, [x0, #32] +; CHECK-NEXT: strh w9, [x1, x10, lsl #1] +; CHECK-NEXT: b LBB0_1 +bb: + %gep = getelementptr inbounds %struct.zot, ptr %arg, i64 0, i32 9 + %l = load i32, ptr %gep, align 4 + %icmp = icmp ugt i32 %l, 2 + %gep1 = getelementptr inbounds %struct.zot, ptr %arg, i64 0, i32 4 + %gep2 = getelementptr inbounds %struct.zot, ptr %arg, i64 0, i32 7 + br label %bb8 + +bb8: ; preds = %bb27, %bb + br i1 %icmp, label %bb9, label %bb27 + +bb9: ; preds = %bb8 + %l10 = load i32, ptr %gep1, align 8 + %gep14 = getelementptr inbounds i32, ptr %p, i64 19 + %l15 = load i32, ptr %gep14, align 1 + %xor = xor i32 %l10, %l15 + %l17 = load i32, ptr %p, align 8 + %and = and i32 %xor, %l17 + store i32 %and, ptr %gep1, align 8 + %zext19 = zext i32 %and to i64 + %gep20 = getelementptr inbounds i16, ptr %p, i64 %zext19 + store i16 10, ptr %gep20, align 2 + br label %bb27 + +bb27: ; preds = %bb9, %bb8 + br label %bb8 +} + +define void @avoid_promotion_2_and(ptr nocapture noundef %arg) { +; CHECK-LABEL: avoid_promotion_2_and: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: add x8, x0, #32 +; CHECK-NEXT: b LBB1_2 +; CHECK-NEXT: LBB1_1: ; %latch +; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: cmp w9, #2 +; CHECK-NEXT: add x8, x8, #56 +; CHECK-NEXT: b.ls LBB1_4 +; CHECK-NEXT: LBB1_2: ; %loop +; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldr w9, [x8, #20] +; CHECK-NEXT: cmp w9, #3 +; CHECK-NEXT: b.lo LBB1_1 +; CHECK-NEXT: ; %bb.3: ; %then +; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: ldp w13, w12, [x8, #12] +; CHECK-NEXT: ldr w10, [x8] +; CHECK-NEXT: ldr x11, [x0] +; CHECK-NEXT: ldr w14, [x8, #8] +; CHECK-NEXT: lsl w10, w10, w13 +; CHECK-NEXT: ldrb w11, [x11, x12] +; CHECK-NEXT: eor w10, w10, w11 +; CHECK-NEXT: ldur w11, [x8, #-24] +; CHECK-NEXT: and x10, x10, x14 +; CHECK-NEXT: ldp x15, x14, [x8, #-16] +; CHECK-NEXT: lsl x13, x10, #1 +; CHECK-NEXT: str w10, [x8] +; CHECK-NEXT: and x10, x11, x12 +; CHECK-NEXT: ldrh w11, [x14, x13] +; CHECK-NEXT: strh w11, [x15, x10, lsl #1] +; CHECK-NEXT: strh w12, [x14, x13] +; CHECK-NEXT: b LBB1_1 +; CHECK-NEXT: LBB1_4: ; %exit +; CHECK-NEXT: ret +entry: + br label %loop + +loop: + %p = phi i64 [ 0, %entry ], [ %p.next, %latch ] + %gep = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 9 + %l = load i32, ptr %gep, align 4 + %icmp = icmp ugt i32 %l, 2 + %gep1 = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 4 + %gep2 = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 7 + %gep3 = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 8 + %gep4 = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 6 + %gep5 = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 3 + %gep6 = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 2 + %gep7 = getelementptr inbounds %struct.zot, ptr %arg, i64 %p, i32 1 + br i1 %icmp, label %then, label %latch + +then: + %l10 = load i32, ptr %gep1, align 8 + %l11 = load i32, ptr %gep2, align 4 + %shl = shl i32 %l10, %l11 + %l12 = load ptr, ptr %arg, align 8 + %l13 = load i32, ptr %gep3, align 8 + %zext = zext i32 %l13 to i64 + %gep14 = getelementptr inbounds i8, ptr %l12, i64 %zext + %l15 = load i8, ptr %gep14, align 1 + %zext16 = zext i8 %l15 to i32 + %xor = xor i32 %shl, %zext16 + %l17 = load i32, ptr %gep4, align 8 + %and = and i32 %xor, %l17 + store i32 %and, ptr %gep1, align 8 + %l18 = load ptr, ptr %gep5, align 8 + %zext19 = zext i32 %and to i64 + %gep20 = getelementptr inbounds i16, ptr %l18, i64 %zext19 + %l21 = load i16, ptr %gep20, align 2 + %l22 = load ptr, ptr %gep6, align 8 + %l23 = load i32, ptr %gep7, align 8 + %and24 = and i32 %l23, %l13 + %zext25 = zext i32 %and24 to i64 + %gep26 = getelementptr inbounds i16, ptr %l22, i64 %zext25 + store i16 %l21, ptr %gep26, align 2 + %trunc = trunc i32 %l13 to i16 + store i16 %trunc, ptr %gep20, align 2 + br label %latch + +latch: + %p.next = add i64 %p, 1 + br i1 %icmp, label %loop, label %exit + +exit: + ret void +} -- GitLab From 1dbdf7658a80088fad1787529df0c33636be1c18 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 5 Jan 2024 13:04:14 +0000 Subject: [PATCH 109/728] [X86] vec_fpext.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/vec_fpext.ll | 232 ++++++++++++++--------------- 1 file changed, 116 insertions(+), 116 deletions(-) diff --git a/llvm/test/CodeGen/X86/vec_fpext.ll b/llvm/test/CodeGen/X86/vec_fpext.ll index 79b5fc09c53e..ca4aa5a1c294 100644 --- a/llvm/test/CodeGen/X86/vec_fpext.ll +++ b/llvm/test/CodeGen/X86/vec_fpext.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse4.1 -show-mc-encoding | FileCheck %s --check-prefixes=SSE,X32-SSE -; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx -show-mc-encoding | FileCheck %s --check-prefixes=AVX,X32-AVX -; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512vl -show-mc-encoding | FileCheck %s --check-prefixes=AVX512VL,X32-AVX512VL +; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse4.1 -show-mc-encoding | FileCheck %s --check-prefixes=SSE,X86-SSE +; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx -show-mc-encoding | FileCheck %s --check-prefixes=AVX,X86-AVX +; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512vl -show-mc-encoding | FileCheck %s --check-prefixes=AVX512VL,X86-AVX512VL ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 -show-mc-encoding | FileCheck %s --check-prefixes=SSE,X64-SSE ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx -show-mc-encoding | FileCheck %s --check-prefixes=AVX,X64-AVX ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl -show-mc-encoding | FileCheck %s --check-prefixes=AVX512VL,X64-AVX512VL @@ -76,29 +76,29 @@ define <4 x double> @fpext_8f32_to_4f64(<8 x float> %a) { ; PR11674 define void @fpext_frommem(ptr %in, ptr %out) { -; X32-SSE-LABEL: fpext_frommem: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-SSE-NEXT: cvtps2pd (%ecx), %xmm0 # encoding: [0x0f,0x5a,0x01] -; X32-SSE-NEXT: movups %xmm0, (%eax) # encoding: [0x0f,0x11,0x00] -; X32-SSE-NEXT: retl # encoding: [0xc3] +; X86-SSE-LABEL: fpext_frommem: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-SSE-NEXT: cvtps2pd (%ecx), %xmm0 # encoding: [0x0f,0x5a,0x01] +; X86-SSE-NEXT: movups %xmm0, (%eax) # encoding: [0x0f,0x11,0x00] +; X86-SSE-NEXT: retl # encoding: [0xc3] ; -; X32-AVX-LABEL: fpext_frommem: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-AVX-NEXT: vcvtps2pd (%ecx), %xmm0 # encoding: [0xc5,0xf8,0x5a,0x01] -; X32-AVX-NEXT: vmovups %xmm0, (%eax) # encoding: [0xc5,0xf8,0x11,0x00] -; X32-AVX-NEXT: retl # encoding: [0xc3] +; X86-AVX-LABEL: fpext_frommem: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-AVX-NEXT: vcvtps2pd (%ecx), %xmm0 # encoding: [0xc5,0xf8,0x5a,0x01] +; X86-AVX-NEXT: vmovups %xmm0, (%eax) # encoding: [0xc5,0xf8,0x11,0x00] +; X86-AVX-NEXT: retl # encoding: [0xc3] ; -; X32-AVX512VL-LABEL: fpext_frommem: -; X32-AVX512VL: # %bb.0: # %entry -; X32-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-AVX512VL-NEXT: vcvtps2pd (%ecx), %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x5a,0x01] -; X32-AVX512VL-NEXT: vmovups %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x00] -; X32-AVX512VL-NEXT: retl # encoding: [0xc3] +; X86-AVX512VL-LABEL: fpext_frommem: +; X86-AVX512VL: # %bb.0: # %entry +; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-AVX512VL-NEXT: vcvtps2pd (%ecx), %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x5a,0x01] +; X86-AVX512VL-NEXT: vmovups %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x00] +; X86-AVX512VL-NEXT: retl # encoding: [0xc3] ; ; X64-SSE-LABEL: fpext_frommem: ; X64-SSE: # %bb.0: # %entry @@ -125,33 +125,33 @@ entry: } define void @fpext_frommem4(ptr %in, ptr %out) { -; X32-SSE-LABEL: fpext_frommem4: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-SSE-NEXT: cvtps2pd (%ecx), %xmm0 # encoding: [0x0f,0x5a,0x01] -; X32-SSE-NEXT: cvtps2pd 8(%ecx), %xmm1 # encoding: [0x0f,0x5a,0x49,0x08] -; X32-SSE-NEXT: movups %xmm1, 16(%eax) # encoding: [0x0f,0x11,0x48,0x10] -; X32-SSE-NEXT: movups %xmm0, (%eax) # encoding: [0x0f,0x11,0x00] -; X32-SSE-NEXT: retl # encoding: [0xc3] +; X86-SSE-LABEL: fpext_frommem4: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-SSE-NEXT: cvtps2pd (%ecx), %xmm0 # encoding: [0x0f,0x5a,0x01] +; X86-SSE-NEXT: cvtps2pd 8(%ecx), %xmm1 # encoding: [0x0f,0x5a,0x49,0x08] +; X86-SSE-NEXT: movups %xmm1, 16(%eax) # encoding: [0x0f,0x11,0x48,0x10] +; X86-SSE-NEXT: movups %xmm0, (%eax) # encoding: [0x0f,0x11,0x00] +; X86-SSE-NEXT: retl # encoding: [0xc3] ; -; X32-AVX-LABEL: fpext_frommem4: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-AVX-NEXT: vcvtps2pd (%ecx), %ymm0 # encoding: [0xc5,0xfc,0x5a,0x01] -; X32-AVX-NEXT: vmovups %ymm0, (%eax) # encoding: [0xc5,0xfc,0x11,0x00] -; X32-AVX-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] -; X32-AVX-NEXT: retl # encoding: [0xc3] +; X86-AVX-LABEL: fpext_frommem4: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-AVX-NEXT: vcvtps2pd (%ecx), %ymm0 # encoding: [0xc5,0xfc,0x5a,0x01] +; X86-AVX-NEXT: vmovups %ymm0, (%eax) # encoding: [0xc5,0xfc,0x11,0x00] +; X86-AVX-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] +; X86-AVX-NEXT: retl # encoding: [0xc3] ; -; X32-AVX512VL-LABEL: fpext_frommem4: -; X32-AVX512VL: # %bb.0: # %entry -; X32-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-AVX512VL-NEXT: vcvtps2pd (%ecx), %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x5a,0x01] -; X32-AVX512VL-NEXT: vmovups %ymm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x00] -; X32-AVX512VL-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] -; X32-AVX512VL-NEXT: retl # encoding: [0xc3] +; X86-AVX512VL-LABEL: fpext_frommem4: +; X86-AVX512VL: # %bb.0: # %entry +; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-AVX512VL-NEXT: vcvtps2pd (%ecx), %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x5a,0x01] +; X86-AVX512VL-NEXT: vmovups %ymm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x00] +; X86-AVX512VL-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] +; X86-AVX512VL-NEXT: retl # encoding: [0xc3] ; ; X64-SSE-LABEL: fpext_frommem4: ; X64-SSE: # %bb.0: # %entry @@ -182,39 +182,39 @@ entry: } define void @fpext_frommem8(ptr %in, ptr %out) { -; X32-SSE-LABEL: fpext_frommem8: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-SSE-NEXT: cvtps2pd 8(%ecx), %xmm0 # encoding: [0x0f,0x5a,0x41,0x08] -; X32-SSE-NEXT: cvtps2pd (%ecx), %xmm1 # encoding: [0x0f,0x5a,0x09] -; X32-SSE-NEXT: cvtps2pd 24(%ecx), %xmm2 # encoding: [0x0f,0x5a,0x51,0x18] -; X32-SSE-NEXT: cvtps2pd 16(%ecx), %xmm3 # encoding: [0x0f,0x5a,0x59,0x10] -; X32-SSE-NEXT: movups %xmm3, 32(%eax) # encoding: [0x0f,0x11,0x58,0x20] -; X32-SSE-NEXT: movups %xmm2, 48(%eax) # encoding: [0x0f,0x11,0x50,0x30] -; X32-SSE-NEXT: movups %xmm1, (%eax) # encoding: [0x0f,0x11,0x08] -; X32-SSE-NEXT: movups %xmm0, 16(%eax) # encoding: [0x0f,0x11,0x40,0x10] -; X32-SSE-NEXT: retl # encoding: [0xc3] +; X86-SSE-LABEL: fpext_frommem8: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-SSE-NEXT: cvtps2pd 8(%ecx), %xmm0 # encoding: [0x0f,0x5a,0x41,0x08] +; X86-SSE-NEXT: cvtps2pd (%ecx), %xmm1 # encoding: [0x0f,0x5a,0x09] +; X86-SSE-NEXT: cvtps2pd 24(%ecx), %xmm2 # encoding: [0x0f,0x5a,0x51,0x18] +; X86-SSE-NEXT: cvtps2pd 16(%ecx), %xmm3 # encoding: [0x0f,0x5a,0x59,0x10] +; X86-SSE-NEXT: movups %xmm3, 32(%eax) # encoding: [0x0f,0x11,0x58,0x20] +; X86-SSE-NEXT: movups %xmm2, 48(%eax) # encoding: [0x0f,0x11,0x50,0x30] +; X86-SSE-NEXT: movups %xmm1, (%eax) # encoding: [0x0f,0x11,0x08] +; X86-SSE-NEXT: movups %xmm0, 16(%eax) # encoding: [0x0f,0x11,0x40,0x10] +; X86-SSE-NEXT: retl # encoding: [0xc3] ; -; X32-AVX-LABEL: fpext_frommem8: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-AVX-NEXT: vcvtps2pd (%ecx), %ymm0 # encoding: [0xc5,0xfc,0x5a,0x01] -; X32-AVX-NEXT: vcvtps2pd 16(%ecx), %ymm1 # encoding: [0xc5,0xfc,0x5a,0x49,0x10] -; X32-AVX-NEXT: vmovups %ymm1, 32(%eax) # encoding: [0xc5,0xfc,0x11,0x48,0x20] -; X32-AVX-NEXT: vmovups %ymm0, (%eax) # encoding: [0xc5,0xfc,0x11,0x00] -; X32-AVX-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] -; X32-AVX-NEXT: retl # encoding: [0xc3] +; X86-AVX-LABEL: fpext_frommem8: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-AVX-NEXT: vcvtps2pd (%ecx), %ymm0 # encoding: [0xc5,0xfc,0x5a,0x01] +; X86-AVX-NEXT: vcvtps2pd 16(%ecx), %ymm1 # encoding: [0xc5,0xfc,0x5a,0x49,0x10] +; X86-AVX-NEXT: vmovups %ymm1, 32(%eax) # encoding: [0xc5,0xfc,0x11,0x48,0x20] +; X86-AVX-NEXT: vmovups %ymm0, (%eax) # encoding: [0xc5,0xfc,0x11,0x00] +; X86-AVX-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] +; X86-AVX-NEXT: retl # encoding: [0xc3] ; -; X32-AVX512VL-LABEL: fpext_frommem8: -; X32-AVX512VL: # %bb.0: # %entry -; X32-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] -; X32-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] -; X32-AVX512VL-NEXT: vcvtps2pd (%ecx), %zmm0 # encoding: [0x62,0xf1,0x7c,0x48,0x5a,0x01] -; X32-AVX512VL-NEXT: vmovups %zmm0, (%eax) # encoding: [0x62,0xf1,0x7c,0x48,0x11,0x00] -; X32-AVX512VL-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] -; X32-AVX512VL-NEXT: retl # encoding: [0xc3] +; X86-AVX512VL-LABEL: fpext_frommem8: +; X86-AVX512VL: # %bb.0: # %entry +; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08] +; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04] +; X86-AVX512VL-NEXT: vcvtps2pd (%ecx), %zmm0 # encoding: [0x62,0xf1,0x7c,0x48,0x5a,0x01] +; X86-AVX512VL-NEXT: vmovups %zmm0, (%eax) # encoding: [0x62,0xf1,0x7c,0x48,0x11,0x00] +; X86-AVX512VL-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77] +; X86-AVX512VL-NEXT: retl # encoding: [0xc3] ; ; X64-SSE-LABEL: fpext_frommem8: ; X64-SSE: # %bb.0: # %entry @@ -251,26 +251,26 @@ entry: } define <2 x double> @fpext_fromconst() { -; X32-SSE-LABEL: fpext_fromconst: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: movaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0] -; X32-SSE-NEXT: # encoding: [0x0f,0x28,0x05,A,A,A,A] -; X32-SSE-NEXT: # fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X32-SSE-NEXT: retl # encoding: [0xc3] +; X86-SSE-LABEL: fpext_fromconst: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: movaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0] +; X86-SSE-NEXT: # encoding: [0x0f,0x28,0x05,A,A,A,A] +; X86-SSE-NEXT: # fixup A - offset: 3, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 +; X86-SSE-NEXT: retl # encoding: [0xc3] ; -; X32-AVX-LABEL: fpext_fromconst: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: vmovaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0] -; X32-AVX-NEXT: # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A] -; X32-AVX-NEXT: # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X32-AVX-NEXT: retl # encoding: [0xc3] +; X86-AVX-LABEL: fpext_fromconst: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: vmovaps {{.*#+}} xmm0 = [1.0E+0,-2.0E+0] +; X86-AVX-NEXT: # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A] +; X86-AVX-NEXT: # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 +; X86-AVX-NEXT: retl # encoding: [0xc3] ; -; X32-AVX512VL-LABEL: fpext_fromconst: -; X32-AVX512VL: # %bb.0: # %entry -; X32-AVX512VL-NEXT: vmovaps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [1.0E+0,-2.0E+0] -; X32-AVX512VL-NEXT: # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A] -; X32-AVX512VL-NEXT: # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 -; X32-AVX512VL-NEXT: retl # encoding: [0xc3] +; X86-AVX512VL-LABEL: fpext_fromconst: +; X86-AVX512VL: # %bb.0: # %entry +; X86-AVX512VL-NEXT: vmovaps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 # EVEX TO VEX Compression xmm0 = [1.0E+0,-2.0E+0] +; X86-AVX512VL-NEXT: # encoding: [0xc5,0xf8,0x28,0x05,A,A,A,A] +; X86-AVX512VL-NEXT: # fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4 +; X86-AVX512VL-NEXT: retl # encoding: [0xc3] ; ; X64-SSE-LABEL: fpext_fromconst: ; X64-SSE: # %bb.0: # %entry @@ -301,26 +301,26 @@ entry: ; Make sure we don't narrow a volatile load. define <2 x double> @PR42079(ptr %x) { -; X32-SSE-LABEL: PR42079: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X32-SSE-NEXT: movaps (%eax), %xmm0 # encoding: [0x0f,0x28,0x00] -; X32-SSE-NEXT: cvtps2pd %xmm0, %xmm0 # encoding: [0x0f,0x5a,0xc0] -; X32-SSE-NEXT: retl # encoding: [0xc3] +; X86-SSE-LABEL: PR42079: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] +; X86-SSE-NEXT: movaps (%eax), %xmm0 # encoding: [0x0f,0x28,0x00] +; X86-SSE-NEXT: cvtps2pd %xmm0, %xmm0 # encoding: [0x0f,0x5a,0xc0] +; X86-SSE-NEXT: retl # encoding: [0xc3] ; -; X32-AVX-LABEL: PR42079: -; X32-AVX: # %bb.0: -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X32-AVX-NEXT: vmovaps (%eax), %xmm0 # encoding: [0xc5,0xf8,0x28,0x00] -; X32-AVX-NEXT: vcvtps2pd %xmm0, %xmm0 # encoding: [0xc5,0xf8,0x5a,0xc0] -; X32-AVX-NEXT: retl # encoding: [0xc3] +; X86-AVX-LABEL: PR42079: +; X86-AVX: # %bb.0: +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] +; X86-AVX-NEXT: vmovaps (%eax), %xmm0 # encoding: [0xc5,0xf8,0x28,0x00] +; X86-AVX-NEXT: vcvtps2pd %xmm0, %xmm0 # encoding: [0xc5,0xf8,0x5a,0xc0] +; X86-AVX-NEXT: retl # encoding: [0xc3] ; -; X32-AVX512VL-LABEL: PR42079: -; X32-AVX512VL: # %bb.0: -; X32-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] -; X32-AVX512VL-NEXT: vmovaps (%eax), %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x00] -; X32-AVX512VL-NEXT: vcvtps2pd %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x5a,0xc0] -; X32-AVX512VL-NEXT: retl # encoding: [0xc3] +; X86-AVX512VL-LABEL: PR42079: +; X86-AVX512VL: # %bb.0: +; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04] +; X86-AVX512VL-NEXT: vmovaps (%eax), %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0x00] +; X86-AVX512VL-NEXT: vcvtps2pd %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x5a,0xc0] +; X86-AVX512VL-NEXT: retl # encoding: [0xc3] ; ; X64-SSE-LABEL: PR42079: ; X64-SSE: # %bb.0: -- GitLab From c3071476609c9b68343a5acf58821aee8ab8ed60 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 5 Jan 2024 13:04:32 +0000 Subject: [PATCH 110/728] [X86] vec_fptrunc.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/vec_fptrunc.ll | 174 +++++++++++++-------------- 1 file changed, 87 insertions(+), 87 deletions(-) diff --git a/llvm/test/CodeGen/X86/vec_fptrunc.ll b/llvm/test/CodeGen/X86/vec_fptrunc.ll index c2f418680e7c..5b2dafaf3ac2 100644 --- a/llvm/test/CodeGen/X86/vec_fptrunc.ll +++ b/llvm/test/CodeGen/X86/vec_fptrunc.ll @@ -1,25 +1,25 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X32-SSE -; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s --check-prefix=X32-AVX +; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X86-SSE +; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s --check-prefix=X86-AVX ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X64-SSE ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=X64-AVX define void @fptrunc_frommem2(ptr %in, ptr %out) { -; X32-SSE-LABEL: fptrunc_frommem2: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-SSE-NEXT: cvtpd2ps (%ecx), %xmm0 -; X32-SSE-NEXT: movlpd %xmm0, (%eax) -; X32-SSE-NEXT: retl -; -; X32-AVX-LABEL: fptrunc_frommem2: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-AVX-NEXT: vcvtpd2psx (%ecx), %xmm0 -; X32-AVX-NEXT: vmovlpd %xmm0, (%eax) -; X32-AVX-NEXT: retl +; X86-SSE-LABEL: fptrunc_frommem2: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE-NEXT: cvtpd2ps (%ecx), %xmm0 +; X86-SSE-NEXT: movlpd %xmm0, (%eax) +; X86-SSE-NEXT: retl +; +; X86-AVX-LABEL: fptrunc_frommem2: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: vcvtpd2psx (%ecx), %xmm0 +; X86-AVX-NEXT: vmovlpd %xmm0, (%eax) +; X86-AVX-NEXT: retl ; ; X64-SSE-LABEL: fptrunc_frommem2: ; X64-SSE: # %bb.0: # %entry @@ -40,23 +40,23 @@ entry: } define void @fptrunc_frommem4(ptr %in, ptr %out) { -; X32-SSE-LABEL: fptrunc_frommem4: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-SSE-NEXT: cvtpd2ps 16(%ecx), %xmm0 -; X32-SSE-NEXT: cvtpd2ps (%ecx), %xmm1 -; X32-SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; X32-SSE-NEXT: movupd %xmm1, (%eax) -; X32-SSE-NEXT: retl -; -; X32-AVX-LABEL: fptrunc_frommem4: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-AVX-NEXT: vcvtpd2psy (%ecx), %xmm0 -; X32-AVX-NEXT: vmovupd %xmm0, (%eax) -; X32-AVX-NEXT: retl +; X86-SSE-LABEL: fptrunc_frommem4: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE-NEXT: cvtpd2ps 16(%ecx), %xmm0 +; X86-SSE-NEXT: cvtpd2ps (%ecx), %xmm1 +; X86-SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; X86-SSE-NEXT: movupd %xmm1, (%eax) +; X86-SSE-NEXT: retl +; +; X86-AVX-LABEL: fptrunc_frommem4: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: vcvtpd2psy (%ecx), %xmm0 +; X86-AVX-NEXT: vmovupd %xmm0, (%eax) +; X86-AVX-NEXT: retl ; ; X64-SSE-LABEL: fptrunc_frommem4: ; X64-SSE: # %bb.0: # %entry @@ -79,29 +79,29 @@ entry: } define void @fptrunc_frommem8(ptr %in, ptr %out) { -; X32-SSE-LABEL: fptrunc_frommem8: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-SSE-NEXT: cvtpd2ps 16(%ecx), %xmm0 -; X32-SSE-NEXT: cvtpd2ps (%ecx), %xmm1 -; X32-SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0] -; X32-SSE-NEXT: cvtpd2ps 48(%ecx), %xmm0 -; X32-SSE-NEXT: cvtpd2ps 32(%ecx), %xmm2 -; X32-SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0] -; X32-SSE-NEXT: movupd %xmm2, 16(%eax) -; X32-SSE-NEXT: movupd %xmm1, (%eax) -; X32-SSE-NEXT: retl -; -; X32-AVX-LABEL: fptrunc_frommem8: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-AVX-NEXT: vcvtpd2psy (%ecx), %xmm0 -; X32-AVX-NEXT: vcvtpd2psy 32(%ecx), %xmm1 -; X32-AVX-NEXT: vmovupd %xmm1, 16(%eax) -; X32-AVX-NEXT: vmovupd %xmm0, (%eax) -; X32-AVX-NEXT: retl +; X86-SSE-LABEL: fptrunc_frommem8: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE-NEXT: cvtpd2ps 16(%ecx), %xmm0 +; X86-SSE-NEXT: cvtpd2ps (%ecx), %xmm1 +; X86-SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; X86-SSE-NEXT: cvtpd2ps 48(%ecx), %xmm0 +; X86-SSE-NEXT: cvtpd2ps 32(%ecx), %xmm2 +; X86-SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0] +; X86-SSE-NEXT: movupd %xmm2, 16(%eax) +; X86-SSE-NEXT: movupd %xmm1, (%eax) +; X86-SSE-NEXT: retl +; +; X86-AVX-LABEL: fptrunc_frommem8: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: vcvtpd2psy (%ecx), %xmm0 +; X86-AVX-NEXT: vcvtpd2psy 32(%ecx), %xmm1 +; X86-AVX-NEXT: vmovupd %xmm1, 16(%eax) +; X86-AVX-NEXT: vmovupd %xmm0, (%eax) +; X86-AVX-NEXT: retl ; ; X64-SSE-LABEL: fptrunc_frommem8: ; X64-SSE: # %bb.0: # %entry @@ -130,17 +130,17 @@ entry: } define <4 x float> @fptrunc_frommem2_zext(ptr %ld) { -; X32-SSE-LABEL: fptrunc_frommem2_zext: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-SSE-NEXT: cvtpd2ps (%eax), %xmm0 -; X32-SSE-NEXT: retl -; -; X32-AVX-LABEL: fptrunc_frommem2_zext: -; X32-AVX: # %bb.0: -; X32-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-AVX-NEXT: vcvtpd2psx (%eax), %xmm0 -; X32-AVX-NEXT: retl +; X86-SSE-LABEL: fptrunc_frommem2_zext: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE-NEXT: cvtpd2ps (%eax), %xmm0 +; X86-SSE-NEXT: retl +; +; X86-AVX-LABEL: fptrunc_frommem2_zext: +; X86-AVX: # %bb.0: +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: vcvtpd2psx (%eax), %xmm0 +; X86-AVX-NEXT: retl ; ; X64-SSE-LABEL: fptrunc_frommem2_zext: ; X64-SSE: # %bb.0: @@ -158,15 +158,15 @@ define <4 x float> @fptrunc_frommem2_zext(ptr %ld) { } define <4 x float> @fptrunc_fromreg2_zext(<2 x double> %arg) { -; X32-SSE-LABEL: fptrunc_fromreg2_zext: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: cvtpd2ps %xmm0, %xmm0 -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: fptrunc_fromreg2_zext: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: cvtpd2ps %xmm0, %xmm0 +; X86-SSE-NEXT: retl ; -; X32-AVX-LABEL: fptrunc_fromreg2_zext: -; X32-AVX: # %bb.0: -; X32-AVX-NEXT: vcvtpd2ps %xmm0, %xmm0 -; X32-AVX-NEXT: retl +; X86-AVX-LABEL: fptrunc_fromreg2_zext: +; X86-AVX: # %bb.0: +; X86-AVX-NEXT: vcvtpd2ps %xmm0, %xmm0 +; X86-AVX-NEXT: retl ; ; X64-SSE-LABEL: fptrunc_fromreg2_zext: ; X64-SSE: # %bb.0: @@ -184,17 +184,17 @@ define <4 x float> @fptrunc_fromreg2_zext(<2 x double> %arg) { ; FIXME: For exact truncations we should be able to fold this. define <4 x float> @fptrunc_fromconst() { -; X32-SSE-LABEL: fptrunc_fromconst: -; X32-SSE: # %bb.0: # %entry -; X32-SSE-NEXT: cvtpd2ps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 -; X32-SSE-NEXT: cvtpd2ps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X32-SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] -; X32-SSE-NEXT: retl -; -; X32-AVX-LABEL: fptrunc_fromconst: -; X32-AVX: # %bb.0: # %entry -; X32-AVX-NEXT: vcvtpd2psy {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X32-AVX-NEXT: retl +; X86-SSE-LABEL: fptrunc_fromconst: +; X86-SSE: # %bb.0: # %entry +; X86-SSE-NEXT: cvtpd2ps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1 +; X86-SSE-NEXT: cvtpd2ps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-SSE-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; X86-SSE-NEXT: retl +; +; X86-AVX-LABEL: fptrunc_fromconst: +; X86-AVX: # %bb.0: # %entry +; X86-AVX-NEXT: vcvtpd2psy {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-AVX-NEXT: retl ; ; X64-SSE-LABEL: fptrunc_fromconst: ; X64-SSE: # %bb.0: # %entry -- GitLab From 4ecd6384af60ea6eb322fc7d3992b31915ec2a5a Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 5 Jan 2024 13:05:06 +0000 Subject: [PATCH 111/728] [X86] fp128-cast.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/fp128-cast.ll | 1096 +++++++++++++-------------- 1 file changed, 548 insertions(+), 548 deletions(-) diff --git a/llvm/test/CodeGen/X86/fp128-cast.ll b/llvm/test/CodeGen/X86/fp128-cast.ll index b39533c0ee21..b77b8f00e44c 100644 --- a/llvm/test/CodeGen/X86/fp128-cast.ll +++ b/llvm/test/CodeGen/X86/fp128-cast.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+sse | FileCheck %s --check-prefix=X64-SSE ; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+sse | FileCheck %s --check-prefix=X64-SSE -; RUN: llc < %s -O2 -mtriple=i686-linux-gnu -mattr=+mmx | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -O2 -mtriple=i686-linux-gnu -mattr=+mmx | FileCheck %s --check-prefix=X86 ; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+avx | FileCheck %s --check-prefix=X64-AVX ; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=+avx | FileCheck %s --check-prefix=X64-AVX ; RUN: llc < %s -O2 -mtriple=x86_64-linux-android -mattr=+avx512f | FileCheck %s --check-prefix=X64-AVX @@ -31,27 +31,27 @@ define dso_local void @TestFPExtF32_F128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPExtF32_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $40, %esp -; X32-NEXT: flds vf32 -; X32-NEXT: fstps {{[0-9]+}}(%esp) -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl %eax, (%esp) -; X32-NEXT: calll __extendsftf2 -; X32-NEXT: subl $4, %esp -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $40, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestFPExtF32_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $40, %esp +; X86-NEXT: flds vf32 +; X86-NEXT: fstps {{[0-9]+}}(%esp) +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, (%esp) +; X86-NEXT: calll __extendsftf2 +; X86-NEXT: subl $4, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $40, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPExtF32_F128: ; X64-AVX: # %bb.0: # %entry @@ -78,27 +78,27 @@ define dso_local void @TestFPExtF64_F128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPExtF64_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $40, %esp -; X32-NEXT: fldl vf64 -; X32-NEXT: fstpl {{[0-9]+}}(%esp) -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl %eax, (%esp) -; X32-NEXT: calll __extenddftf2 -; X32-NEXT: subl $4, %esp -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $40, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestFPExtF64_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $40, %esp +; X86-NEXT: fldl vf64 +; X86-NEXT: fstpl {{[0-9]+}}(%esp) +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, (%esp) +; X86-NEXT: calll __extenddftf2 +; X86-NEXT: subl $4, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $40, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPExtF64_F128: ; X64-AVX: # %bb.0: # %entry @@ -126,27 +126,27 @@ define dso_local void @TestFPExtF80_F128() nounwind { ; X64-SSE-NEXT: addq $24, %rsp ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPExtF80_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $40, %esp -; X32-NEXT: fldt vf80 -; X32-NEXT: fstpt {{[0-9]+}}(%esp) -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl %eax, (%esp) -; X32-NEXT: calll __extendxftf2 -; X32-NEXT: subl $4, %esp -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $40, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestFPExtF80_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $40, %esp +; X86-NEXT: fldt vf80 +; X86-NEXT: fstpt {{[0-9]+}}(%esp) +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, (%esp) +; X86-NEXT: calll __extendxftf2 +; X86-NEXT: subl $4, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $40, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPExtF80_F128: ; X64-AVX: # %bb.0: # %entry @@ -174,18 +174,18 @@ define dso_local void @TestFPToSIF128_I16() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToSIF128_I16: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __fixtfsi -; X32-NEXT: addl $16, %esp -; X32-NEXT: movw %ax, vi16 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPToSIF128_I16: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __fixtfsi +; X86-NEXT: addl $16, %esp +; X86-NEXT: movw %ax, vi16 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToSIF128_I16: ; X64-AVX: # %bb.0: # %entry @@ -212,18 +212,18 @@ define dso_local void @TestFPToUIF128_I16() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToUIF128_I16: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __fixunstfsi -; X32-NEXT: addl $16, %esp -; X32-NEXT: movw %ax, vi16 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPToUIF128_I16: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __fixunstfsi +; X86-NEXT: addl $16, %esp +; X86-NEXT: movw %ax, vi16 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToUIF128_I16: ; X64-AVX: # %bb.0: # %entry @@ -250,18 +250,18 @@ define dso_local void @TestFPToSIF128_I32() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToSIF128_I32: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __fixtfsi -; X32-NEXT: addl $16, %esp -; X32-NEXT: movl %eax, vi32 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPToSIF128_I32: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __fixtfsi +; X86-NEXT: addl $16, %esp +; X86-NEXT: movl %eax, vi32 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToSIF128_I32: ; X64-AVX: # %bb.0: # %entry @@ -288,18 +288,18 @@ define dso_local void @TestFPToUIF128_U32() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToUIF128_U32: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __fixunstfsi -; X32-NEXT: addl $16, %esp -; X32-NEXT: movl %eax, vu32 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPToUIF128_U32: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __fixunstfsi +; X86-NEXT: addl $16, %esp +; X86-NEXT: movl %eax, vu32 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToUIF128_U32: ; X64-AVX: # %bb.0: # %entry @@ -327,20 +327,20 @@ define dso_local void @TestFPToSIF128_I64() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToSIF128_I64: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __fixtfsi -; X32-NEXT: addl $16, %esp -; X32-NEXT: movl %eax, vi64 -; X32-NEXT: sarl $31, %eax -; X32-NEXT: movl %eax, vi64+4 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPToSIF128_I64: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __fixtfsi +; X86-NEXT: addl $16, %esp +; X86-NEXT: movl %eax, vi64 +; X86-NEXT: sarl $31, %eax +; X86-NEXT: movl %eax, vi64+4 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToSIF128_I64: ; X64-AVX: # %bb.0: # %entry @@ -370,19 +370,19 @@ define dso_local void @TestFPToUIF128_U64() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToUIF128_U64: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __fixunstfsi -; X32-NEXT: addl $16, %esp -; X32-NEXT: movl %eax, vu64 -; X32-NEXT: movl $0, vu64+4 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPToUIF128_U64: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __fixunstfsi +; X86-NEXT: addl $16, %esp +; X86-NEXT: movl %eax, vu64 +; X86-NEXT: movl $0, vu64+4 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToUIF128_U64: ; X64-AVX: # %bb.0: # %entry @@ -412,29 +412,29 @@ define dso_local void @TestFPToSIF128_I128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToSIF128_I128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $36, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __fixtfti -; X32-NEXT: addl $28, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vi128+12 -; X32-NEXT: movl %edx, vi128+8 -; X32-NEXT: movl %ecx, vi128+4 -; X32-NEXT: movl %eax, vi128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestFPToSIF128_I128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $36, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __fixtfti +; X86-NEXT: addl $28, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vi128+12 +; X86-NEXT: movl %edx, vi128+8 +; X86-NEXT: movl %ecx, vi128+4 +; X86-NEXT: movl %eax, vi128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToSIF128_I128: ; X64-AVX: # %bb.0: # %entry @@ -463,29 +463,29 @@ define dso_local void @TestFPToUIF128_U128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPToUIF128_U128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $36, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __fixunstfti -; X32-NEXT: addl $28, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vu128+12 -; X32-NEXT: movl %edx, vu128+8 -; X32-NEXT: movl %ecx, vu128+4 -; X32-NEXT: movl %eax, vu128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestFPToUIF128_U128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $36, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __fixunstfti +; X86-NEXT: addl $28, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vu128+12 +; X86-NEXT: movl %edx, vu128+8 +; X86-NEXT: movl %ecx, vu128+4 +; X86-NEXT: movl %eax, vu128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPToUIF128_U128: ; X64-AVX: # %bb.0: # %entry @@ -513,18 +513,18 @@ define dso_local void @TestFPTruncF128_F32() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPTruncF128_F32: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __trunctfsf2 -; X32-NEXT: addl $16, %esp -; X32-NEXT: fstps vf32 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPTruncF128_F32: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __trunctfsf2 +; X86-NEXT: addl $16, %esp +; X86-NEXT: fstps vf32 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPTruncF128_F32: ; X64-AVX: # %bb.0: # %entry @@ -551,18 +551,18 @@ define dso_local void @TestFPTruncF128_F64() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPTruncF128_F64: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __trunctfdf2 -; X32-NEXT: addl $16, %esp -; X32-NEXT: fstpl vf64 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPTruncF128_F64: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __trunctfdf2 +; X86-NEXT: addl $16, %esp +; X86-NEXT: fstpl vf64 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPTruncF128_F64: ; X64-AVX: # %bb.0: # %entry @@ -589,18 +589,18 @@ define dso_local void @TestFPTruncF128_F80() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestFPTruncF128_F80: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl vf128+12 -; X32-NEXT: pushl vf128+8 -; X32-NEXT: pushl vf128+4 -; X32-NEXT: pushl vf128 -; X32-NEXT: calll __trunctfxf2 -; X32-NEXT: addl $16, %esp -; X32-NEXT: fstpt vf80 -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestFPTruncF128_F80: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl vf128+12 +; X86-NEXT: pushl vf128+8 +; X86-NEXT: pushl vf128+4 +; X86-NEXT: pushl vf128 +; X86-NEXT: calll __trunctfxf2 +; X86-NEXT: addl $16, %esp +; X86-NEXT: fstpt vf80 +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestFPTruncF128_F80: ; X64-AVX: # %bb.0: # %entry @@ -627,28 +627,28 @@ define dso_local void @TestSIToFPI16_F128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestSIToFPI16_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $24, %esp -; X32-NEXT: movswl vi16, %eax -; X32-NEXT: subl $8, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %ecx -; X32-NEXT: pushl %eax -; X32-NEXT: pushl %ecx -; X32-NEXT: calll __floatsitf -; X32-NEXT: addl $12, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestSIToFPI16_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $24, %esp +; X86-NEXT: movswl vi16, %eax +; X86-NEXT: subl $8, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %ecx +; X86-NEXT: pushl %eax +; X86-NEXT: pushl %ecx +; X86-NEXT: calll __floatsitf +; X86-NEXT: addl $12, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestSIToFPI16_F128: ; X64-AVX: # %bb.0: # %entry @@ -675,28 +675,28 @@ define dso_local void @TestSIToFPU16_F128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestSIToFPU16_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $24, %esp -; X32-NEXT: movzwl vi16, %eax -; X32-NEXT: subl $8, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %ecx -; X32-NEXT: pushl %eax -; X32-NEXT: pushl %ecx -; X32-NEXT: calll __floatunsitf -; X32-NEXT: addl $12, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestSIToFPU16_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $24, %esp +; X86-NEXT: movzwl vi16, %eax +; X86-NEXT: subl $8, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %ecx +; X86-NEXT: pushl %eax +; X86-NEXT: pushl %ecx +; X86-NEXT: calll __floatunsitf +; X86-NEXT: addl $12, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestSIToFPU16_F128: ; X64-AVX: # %bb.0: # %entry @@ -723,26 +723,26 @@ define dso_local void @TestSIToFPI32_F128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestSIToFPI32_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $32, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vi32 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __floatsitf -; X32-NEXT: addl $12, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestSIToFPI32_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $32, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vi32 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __floatsitf +; X86-NEXT: addl $12, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestSIToFPI32_F128: ; X64-AVX: # %bb.0: # %entry @@ -769,26 +769,26 @@ define dso_local void @TestUIToFPU32_F128() #2 { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestUIToFPU32_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $32, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vu32 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __floatunsitf -; X32-NEXT: addl $12, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestUIToFPU32_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $32, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vu32 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __floatunsitf +; X86-NEXT: addl $12, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestUIToFPU32_F128: ; X64-AVX: # %bb.0: # %entry @@ -815,27 +815,27 @@ define dso_local void @TestSIToFPI64_F128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestSIToFPI64_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $28, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vi64+4 -; X32-NEXT: pushl vi64 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __floatditf -; X32-NEXT: addl $12, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestSIToFPI64_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $28, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vi64+4 +; X86-NEXT: pushl vi64 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __floatditf +; X86-NEXT: addl $12, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestSIToFPI64_F128: ; X64-AVX: # %bb.0: # %entry @@ -862,27 +862,27 @@ define dso_local void @TestUIToFPU64_F128() #2 { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestUIToFPU64_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $28, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vu64+4 -; X32-NEXT: pushl vu64 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __floatunditf -; X32-NEXT: addl $12, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestUIToFPU64_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $28, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vu64+4 +; X86-NEXT: pushl vu64 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __floatunditf +; X86-NEXT: addl $12, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestUIToFPU64_F128: ; X64-AVX: # %bb.0: # %entry @@ -910,29 +910,29 @@ define dso_local void @TestSIToFPI128_F128() nounwind { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestSIToFPI128_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $36, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vi128+12 -; X32-NEXT: pushl vi128+8 -; X32-NEXT: pushl vi128+4 -; X32-NEXT: pushl vi128 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __floattitf -; X32-NEXT: addl $28, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestSIToFPI128_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $36, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vi128+12 +; X86-NEXT: pushl vi128+8 +; X86-NEXT: pushl vi128+4 +; X86-NEXT: pushl vi128 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __floattitf +; X86-NEXT: addl $28, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestSIToFPI128_F128: ; X64-AVX: # %bb.0: # %entry @@ -961,29 +961,29 @@ define dso_local void @TestUIToFPU128_F128() #2 { ; X64-SSE-NEXT: popq %rax ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestUIToFPU128_F128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: subl $36, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: pushl vu128+12 -; X32-NEXT: pushl vu128+8 -; X32-NEXT: pushl vu128+4 -; X32-NEXT: pushl vu128 -; X32-NEXT: pushl %eax -; X32-NEXT: calll __floatuntitf -; X32-NEXT: addl $28, %esp -; X32-NEXT: movl (%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl %esi, vf128+12 -; X32-NEXT: movl %edx, vf128+8 -; X32-NEXT: movl %ecx, vf128+4 -; X32-NEXT: movl %eax, vf128 -; X32-NEXT: addl $24, %esp -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: TestUIToFPU128_F128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: subl $36, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: pushl vu128+12 +; X86-NEXT: pushl vu128+8 +; X86-NEXT: pushl vu128+4 +; X86-NEXT: pushl vu128 +; X86-NEXT: pushl %eax +; X86-NEXT: calll __floatuntitf +; X86-NEXT: addl $28, %esp +; X86-NEXT: movl (%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl %esi, vf128+12 +; X86-NEXT: movl %edx, vf128+8 +; X86-NEXT: movl %ecx, vf128+4 +; X86-NEXT: movl %eax, vf128 +; X86-NEXT: addl $24, %esp +; X86-NEXT: popl %esi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestUIToFPU128_F128: ; X64-AVX: # %bb.0: # %entry @@ -1014,25 +1014,25 @@ define dso_local i32 @TestConst128(fp128 %v) nounwind { ; X64-SSE-NEXT: popq %rcx ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestConst128: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl $1073676288 # imm = 0x3FFF0000 -; X32-NEXT: pushl $0 -; X32-NEXT: pushl $0 -; X32-NEXT: pushl $0 -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: calll __gttf2 -; X32-NEXT: addl $32, %esp -; X32-NEXT: xorl %ecx, %ecx -; X32-NEXT: testl %eax, %eax -; X32-NEXT: setg %cl -; X32-NEXT: movl %ecx, %eax -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestConst128: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl $1073676288 # imm = 0x3FFF0000 +; X86-NEXT: pushl $0 +; X86-NEXT: pushl $0 +; X86-NEXT: pushl $0 +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: calll __gttf2 +; X86-NEXT: addl $32, %esp +; X86-NEXT: xorl %ecx, %ecx +; X86-NEXT: testl %eax, %eax +; X86-NEXT: setg %cl +; X86-NEXT: movl %ecx, %eax +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestConst128: ; X64-AVX: # %bb.0: # %entry @@ -1065,25 +1065,25 @@ define dso_local i32 @TestConst128Zero(fp128 %v) nounwind { ; X64-SSE-NEXT: popq %rcx ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestConst128Zero: -; X32: # %bb.0: # %entry -; X32-NEXT: subl $12, %esp -; X32-NEXT: pushl $0 -; X32-NEXT: pushl $0 -; X32-NEXT: pushl $0 -; X32-NEXT: pushl $0 -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: pushl {{[0-9]+}}(%esp) -; X32-NEXT: calll __gttf2 -; X32-NEXT: addl $32, %esp -; X32-NEXT: xorl %ecx, %ecx -; X32-NEXT: testl %eax, %eax -; X32-NEXT: setg %cl -; X32-NEXT: movl %ecx, %eax -; X32-NEXT: addl $12, %esp -; X32-NEXT: retl +; X86-LABEL: TestConst128Zero: +; X86: # %bb.0: # %entry +; X86-NEXT: subl $12, %esp +; X86-NEXT: pushl $0 +; X86-NEXT: pushl $0 +; X86-NEXT: pushl $0 +; X86-NEXT: pushl $0 +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: pushl {{[0-9]+}}(%esp) +; X86-NEXT: calll __gttf2 +; X86-NEXT: addl $32, %esp +; X86-NEXT: xorl %ecx, %ecx +; X86-NEXT: testl %eax, %eax +; X86-NEXT: setg %cl +; X86-NEXT: movl %ecx, %eax +; X86-NEXT: addl $12, %esp +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestConst128Zero: ; X64-AVX: # %bb.0: # %entry @@ -1132,36 +1132,36 @@ define dso_local i32 @TestBits128(fp128 %ld) nounwind { ; X64-SSE-NEXT: addq $24, %rsp ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestBits128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %edi -; X32-NEXT: pushl %esi -; X32-NEXT: subl $20, %esp -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %edi -; X32-NEXT: subl $12, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %edx -; X32-NEXT: pushl %edi -; X32-NEXT: pushl %esi -; X32-NEXT: pushl %ecx -; X32-NEXT: pushl %eax -; X32-NEXT: pushl %edi -; X32-NEXT: pushl %esi -; X32-NEXT: pushl %ecx -; X32-NEXT: pushl %eax -; X32-NEXT: pushl %edx -; X32-NEXT: calll __multf3 -; X32-NEXT: addl $44, %esp -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: xorl %eax, %eax -; X32-NEXT: orl (%esp), %ecx -; X32-NEXT: sete %al -; X32-NEXT: addl $20, %esp -; X32-NEXT: popl %esi -; X32-NEXT: popl %edi -; X32-NEXT: retl +; X86-LABEL: TestBits128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: subl $20, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: subl $12, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %edx +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: pushl %ecx +; X86-NEXT: pushl %eax +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: pushl %ecx +; X86-NEXT: pushl %eax +; X86-NEXT: pushl %edx +; X86-NEXT: calll __multf3 +; X86-NEXT: addl $44, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: xorl %eax, %eax +; X86-NEXT: orl (%esp), %ecx +; X86-NEXT: sete %al +; X86-NEXT: addl $20, %esp +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl ; ; X64-AVX-LABEL: TestBits128: ; X64-AVX: # %bb.0: # %entry @@ -1212,26 +1212,26 @@ define fp128 @TestPair128(i64 %a, i64 %b) nounwind { ; X64-SSE-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0 ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestPair128: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %edi -; X32-NEXT: pushl %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %edi -; X32-NEXT: addl $3, %ecx -; X32-NEXT: adcl $0, %edx -; X32-NEXT: adcl $0, %esi -; X32-NEXT: adcl $0, %edi -; X32-NEXT: movl %esi, 8(%eax) -; X32-NEXT: movl %edx, 4(%eax) -; X32-NEXT: movl %ecx, (%eax) -; X32-NEXT: movl %edi, 12(%eax) -; X32-NEXT: popl %esi -; X32-NEXT: popl %edi -; X32-NEXT: retl $4 +; X86-LABEL: TestPair128: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: addl $3, %ecx +; X86-NEXT: adcl $0, %edx +; X86-NEXT: adcl $0, %esi +; X86-NEXT: adcl $0, %edi +; X86-NEXT: movl %esi, 8(%eax) +; X86-NEXT: movl %edx, 4(%eax) +; X86-NEXT: movl %ecx, (%eax) +; X86-NEXT: movl %edi, 12(%eax) +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl $4 ; ; X64-AVX-LABEL: TestPair128: ; X64-AVX: # %bb.0: # %entry @@ -1266,55 +1266,55 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind { ; X64-SSE-NEXT: .LBB26_2: # %cleanup ; X64-SSE-NEXT: retq ; -; X32-LABEL: TestTruncCopysign: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %edi -; X32-NEXT: pushl %esi -; X32-NEXT: subl $36, %esp -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edi -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: cmpl $50001, {{[0-9]+}}(%esp) # imm = 0xC351 -; X32-NEXT: jl .LBB26_4 -; X32-NEXT: # %bb.1: # %if.then -; X32-NEXT: pushl %eax -; X32-NEXT: pushl %ecx -; X32-NEXT: pushl %edi -; X32-NEXT: pushl %edx -; X32-NEXT: calll __trunctfdf2 -; X32-NEXT: addl $16, %esp -; X32-NEXT: fstpl {{[0-9]+}}(%esp) -; X32-NEXT: testb $-128, {{[0-9]+}}(%esp) -; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} -; X32-NEXT: jne .LBB26_3 -; X32-NEXT: # %bb.2: # %if.then -; X32-NEXT: fstp %st(1) -; X32-NEXT: fldz -; X32-NEXT: .LBB26_3: # %if.then -; X32-NEXT: fstp %st(0) -; X32-NEXT: subl $16, %esp -; X32-NEXT: leal {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl %eax, (%esp) -; X32-NEXT: fstpl {{[0-9]+}}(%esp) -; X32-NEXT: calll __extenddftf2 -; X32-NEXT: addl $12, %esp -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl (%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edi -; X32-NEXT: .LBB26_4: # %cleanup -; X32-NEXT: movl %edx, (%esi) -; X32-NEXT: movl %edi, 4(%esi) -; X32-NEXT: movl %ecx, 8(%esi) -; X32-NEXT: movl %eax, 12(%esi) -; X32-NEXT: movl %esi, %eax -; X32-NEXT: addl $36, %esp -; X32-NEXT: popl %esi -; X32-NEXT: popl %edi -; X32-NEXT: retl $4 +; X86-LABEL: TestTruncCopysign: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: subl $36, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: cmpl $50001, {{[0-9]+}}(%esp) # imm = 0xC351 +; X86-NEXT: jl .LBB26_4 +; X86-NEXT: # %bb.1: # %if.then +; X86-NEXT: pushl %eax +; X86-NEXT: pushl %ecx +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %edx +; X86-NEXT: calll __trunctfdf2 +; X86-NEXT: addl $16, %esp +; X86-NEXT: fstpl {{[0-9]+}}(%esp) +; X86-NEXT: testb $-128, {{[0-9]+}}(%esp) +; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; X86-NEXT: jne .LBB26_3 +; X86-NEXT: # %bb.2: # %if.then +; X86-NEXT: fstp %st(1) +; X86-NEXT: fldz +; X86-NEXT: .LBB26_3: # %if.then +; X86-NEXT: fstp %st(0) +; X86-NEXT: subl $16, %esp +; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, (%esp) +; X86-NEXT: fstpl {{[0-9]+}}(%esp) +; X86-NEXT: calll __extenddftf2 +; X86-NEXT: addl $12, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl (%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: .LBB26_4: # %cleanup +; X86-NEXT: movl %edx, (%esi) +; X86-NEXT: movl %edi, 4(%esi) +; X86-NEXT: movl %ecx, 8(%esi) +; X86-NEXT: movl %eax, 12(%esi) +; X86-NEXT: movl %esi, %eax +; X86-NEXT: addl $36, %esp +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl $4 ; ; X64-AVX-LABEL: TestTruncCopysign: ; X64-AVX: # %bb.0: # %entry @@ -1357,15 +1357,15 @@ define i1 @PR34866(i128 %x) nounwind { ; X64-SSE-NEXT: sete %al ; X64-SSE-NEXT: retq ; -; X32-LABEL: PR34866: -; X32: # %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: orl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: orl {{[0-9]+}}(%esp), %eax -; X32-NEXT: orl %ecx, %eax -; X32-NEXT: sete %al -; X32-NEXT: retl +; X86-LABEL: PR34866: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: orl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl %ecx, %eax +; X86-NEXT: sete %al +; X86-NEXT: retl ; ; X64-AVX-LABEL: PR34866: ; X64-AVX: # %bb.0: @@ -1392,15 +1392,15 @@ define i1 @PR34866_commute(i128 %x) nounwind { ; X64-SSE-NEXT: sete %al ; X64-SSE-NEXT: retq ; -; X32-LABEL: PR34866_commute: -; X32: # %bb.0: -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: orl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: orl {{[0-9]+}}(%esp), %eax -; X32-NEXT: orl %ecx, %eax -; X32-NEXT: sete %al -; X32-NEXT: retl +; X86-LABEL: PR34866_commute: +; X86: # %bb.0: +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: orl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: orl {{[0-9]+}}(%esp), %eax +; X86-NEXT: orl %ecx, %eax +; X86-NEXT: sete %al +; X86-NEXT: retl ; ; X64-AVX-LABEL: PR34866_commute: ; X64-AVX: # %bb.0: -- GitLab From b51130a3311d5553c92408e6e52ea74269fa5a0b Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 5 Jan 2024 13:05:38 +0000 Subject: [PATCH 112/728] [X86] combine-fneg.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/combine-fneg.ll | 136 +++++++++++++------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/llvm/test/CodeGen/X86/combine-fneg.ll b/llvm/test/CodeGen/X86/combine-fneg.ll index df7dcc0f1528..e4a07348dc96 100644 --- a/llvm/test/CodeGen/X86/combine-fneg.ll +++ b/llvm/test/CodeGen/X86/combine-fneg.ll @@ -1,17 +1,17 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse | FileCheck %s --check-prefix=X32-SSE --check-prefix=X32-SSE1 -; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE --check-prefix=X32-SSE2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2 | FileCheck %s --check-prefix=X64-SSE --check-prefix=X64-SSE1 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X64-SSE --check-prefix=X64-SSE2 +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse | FileCheck %s --check-prefixes=X86-SSE,X86-SSE1 +; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X86-SSE,X86-SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2 | FileCheck %s --check-prefixes=X64-SSE,X64-SSE1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X64-SSE,X64-SSE2 ; FNEG is defined as subtraction from -0.0. ; This test verifies that we use an xor with a constant to flip the sign bits; no subtraction needed. define <4 x float> @t1(<4 x float> %Q) nounwind { -; X32-SSE-LABEL: t1: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: t1: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: t1: ; X64-SSE: # %bb.0: @@ -24,10 +24,10 @@ define <4 x float> @t1(<4 x float> %Q) nounwind { ; Possibly misplaced test, but since we're checking undef scenarios... define float @scalar_fsub_neg0_undef(float %x) nounwind { -; X32-SSE-LABEL: scalar_fsub_neg0_undef: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: fldz -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: scalar_fsub_neg0_undef: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: fldz +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: scalar_fsub_neg0_undef: ; X64-SSE: # %bb.0: @@ -37,10 +37,10 @@ define float @scalar_fsub_neg0_undef(float %x) nounwind { } define float @scalar_fneg_undef(float %x) nounwind { -; X32-SSE-LABEL: scalar_fneg_undef: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: fldz -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: scalar_fneg_undef: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: fldz +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: scalar_fneg_undef: ; X64-SSE: # %bb.0: @@ -50,9 +50,9 @@ define float @scalar_fneg_undef(float %x) nounwind { } define <4 x float> @fsub_neg0_undef(<4 x float> %Q) nounwind { -; X32-SSE-LABEL: fsub_neg0_undef: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: fsub_neg0_undef: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: fsub_neg0_undef: ; X64-SSE: # %bb.0: @@ -62,9 +62,9 @@ define <4 x float> @fsub_neg0_undef(<4 x float> %Q) nounwind { } define <4 x float> @fneg_undef(<4 x float> %Q) nounwind { -; X32-SSE-LABEL: fneg_undef: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: fneg_undef: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: fneg_undef: ; X64-SSE: # %bb.0: @@ -74,9 +74,9 @@ define <4 x float> @fneg_undef(<4 x float> %Q) nounwind { } define <4 x float> @fsub_neg0_undef_elts_undef(<4 x float> %x) { -; X32-SSE-LABEL: fsub_neg0_undef_elts_undef: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: fsub_neg0_undef_elts_undef: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: fsub_neg0_undef_elts_undef: ; X64-SSE: # %bb.0: @@ -87,12 +87,12 @@ define <4 x float> @fsub_neg0_undef_elts_undef(<4 x float> %x) { ; This test verifies that we generate an FP subtraction because "0.0 - x" is not an fneg. define <4 x float> @t2(<4 x float> %Q) nounwind { -; X32-SSE-LABEL: t2: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: xorps %xmm1, %xmm1 -; X32-SSE-NEXT: subps %xmm0, %xmm1 -; X32-SSE-NEXT: movaps %xmm1, %xmm0 -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: t2: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: xorps %xmm1, %xmm1 +; X86-SSE-NEXT: subps %xmm0, %xmm1 +; X86-SSE-NEXT: movaps %xmm1, %xmm0 +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: t2: ; X64-SSE: # %bb.0: @@ -116,33 +116,33 @@ define <4 x float> @t2(<4 x float> %Q) nounwind { ; movd (move to xmm return register) define <2 x float> @fneg_bitcast(i64 %i) nounwind { -; X32-SSE1-LABEL: fneg_bitcast: -; X32-SSE1: # %bb.0: -; X32-SSE1-NEXT: pushl %ebp -; X32-SSE1-NEXT: movl %esp, %ebp -; X32-SSE1-NEXT: andl $-16, %esp -; X32-SSE1-NEXT: subl $16, %esp -; X32-SSE1-NEXT: movl $-2147483648, %eax # imm = 0x80000000 -; X32-SSE1-NEXT: movl 12(%ebp), %ecx -; X32-SSE1-NEXT: xorl %eax, %ecx -; X32-SSE1-NEXT: movl %ecx, {{[0-9]+}}(%esp) -; X32-SSE1-NEXT: xorl 8(%ebp), %eax -; X32-SSE1-NEXT: movl %eax, (%esp) -; X32-SSE1-NEXT: movaps (%esp), %xmm0 -; X32-SSE1-NEXT: movl %ebp, %esp -; X32-SSE1-NEXT: popl %ebp -; X32-SSE1-NEXT: retl +; X86-SSE1-LABEL: fneg_bitcast: +; X86-SSE1: # %bb.0: +; X86-SSE1-NEXT: pushl %ebp +; X86-SSE1-NEXT: movl %esp, %ebp +; X86-SSE1-NEXT: andl $-16, %esp +; X86-SSE1-NEXT: subl $16, %esp +; X86-SSE1-NEXT: movl $-2147483648, %eax # imm = 0x80000000 +; X86-SSE1-NEXT: movl 12(%ebp), %ecx +; X86-SSE1-NEXT: xorl %eax, %ecx +; X86-SSE1-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; X86-SSE1-NEXT: xorl 8(%ebp), %eax +; X86-SSE1-NEXT: movl %eax, (%esp) +; X86-SSE1-NEXT: movaps (%esp), %xmm0 +; X86-SSE1-NEXT: movl %ebp, %esp +; X86-SSE1-NEXT: popl %ebp +; X86-SSE1-NEXT: retl ; -; X32-SSE2-LABEL: fneg_bitcast: -; X32-SSE2: # %bb.0: -; X32-SSE2-NEXT: movl $-2147483648, %eax # imm = 0x80000000 -; X32-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-SSE2-NEXT: xorl %eax, %ecx -; X32-SSE2-NEXT: movd %ecx, %xmm1 -; X32-SSE2-NEXT: xorl {{[0-9]+}}(%esp), %eax -; X32-SSE2-NEXT: movd %eax, %xmm0 -; X32-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; X32-SSE2-NEXT: retl +; X86-SSE2-LABEL: fneg_bitcast: +; X86-SSE2: # %bb.0: +; X86-SSE2-NEXT: movl $-2147483648, %eax # imm = 0x80000000 +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE2-NEXT: xorl %eax, %ecx +; X86-SSE2-NEXT: movd %ecx, %xmm1 +; X86-SSE2-NEXT: xorl {{[0-9]+}}(%esp), %eax +; X86-SSE2-NEXT: movd %eax, %xmm0 +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X86-SSE2-NEXT: retl ; ; X64-SSE1-LABEL: fneg_bitcast: ; X64-SSE1: # %bb.0: @@ -164,10 +164,10 @@ define <2 x float> @fneg_bitcast(i64 %i) nounwind { } define <4 x float> @fneg_undef_elts_v4f32(<4 x float> %x) { -; X32-SSE-LABEL: fneg_undef_elts_v4f32: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: fneg_undef_elts_v4f32: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: fneg_undef_elts_v4f32: ; X64-SSE: # %bb.0: @@ -180,9 +180,9 @@ define <4 x float> @fneg_undef_elts_v4f32(<4 x float> %x) { ; This isn't fneg, but similarly check that (X - 0.0) is simplified. define <4 x float> @fsub0_undef_elts_v4f32(<4 x float> %x) { -; X32-SSE-LABEL: fsub0_undef_elts_v4f32: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: fsub0_undef_elts_v4f32: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: fsub0_undef_elts_v4f32: ; X64-SSE: # %bb.0: @@ -192,10 +192,10 @@ define <4 x float> @fsub0_undef_elts_v4f32(<4 x float> %x) { } define <4 x float> @fneg(<4 x float> %Q) nounwind { -; X32-SSE-LABEL: fneg: -; X32-SSE: # %bb.0: -; X32-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X32-SSE-NEXT: retl +; X86-SSE-LABEL: fneg: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: xorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-SSE-NEXT: retl ; ; X64-SSE-LABEL: fneg: ; X64-SSE: # %bb.0: -- GitLab From ae81400a0fad6040483f1af5947388aecca5c3f7 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 5 Jan 2024 13:06:36 +0000 Subject: [PATCH 113/728] [X86] keylocker-intrinsics.ll - replace X32 checks with X86. NFC. We try to use X32 for gnux32 triples only. --- llvm/test/CodeGen/X86/keylocker-intrinsics.ll | 480 +++++++++--------- 1 file changed, 240 insertions(+), 240 deletions(-) diff --git a/llvm/test/CodeGen/X86/keylocker-intrinsics.ll b/llvm/test/CodeGen/X86/keylocker-intrinsics.ll index 3892d84e57db..f01411c64acb 100644 --- a/llvm/test/CodeGen/X86/keylocker-intrinsics.ll +++ b/llvm/test/CodeGen/X86/keylocker-intrinsics.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unkown-unknown -mattr=+kl,widekl | FileCheck %s --check-prefix=X64 -; RUN: llc < %s -verify-machineinstrs -mtriple=i386-unkown-unknown -mattr=+kl,widekl -mattr=+avx2 | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -verify-machineinstrs -mtriple=i386-unkown-unknown -mattr=+kl,widekl -mattr=+avx2 | FileCheck %s --check-prefix=X86 ; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unkown-unknown -mattr=+widekl | FileCheck %s --check-prefix=X64 -; RUN: llc < %s -verify-machineinstrs -mtriple=i386-unkown-unknown -mattr=+widekl -mattr=+avx2 | FileCheck %s --check-prefix=X32 +; RUN: llc < %s -verify-machineinstrs -mtriple=i386-unkown-unknown -mattr=+widekl -mattr=+avx2 | FileCheck %s --check-prefix=X86 declare void @llvm.x86.loadiwkey(<2 x i64>, <2 x i64>, <2 x i64>, i32) declare { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32, <2 x i64>) @@ -23,11 +23,11 @@ define void @test_loadiwkey(i32 %ctl, <2 x i64> %intkey, <2 x i64> %enkey_lo, <2 ; X64-NEXT: loadiwkey %xmm2, %xmm1 ; X64-NEXT: retq ; -; X32-LABEL: test_loadiwkey: -; X32: # %bb.0: # %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: loadiwkey %xmm2, %xmm1 -; X32-NEXT: retl +; X86-LABEL: test_loadiwkey: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: loadiwkey %xmm2, %xmm1 +; X86-NEXT: retl entry: tail call void @llvm.x86.loadiwkey(<2 x i64> %intkey, <2 x i64> %enkey_lo, <2 x i64> %enkey_hi, i32 %ctl) ret void @@ -42,19 +42,19 @@ define i32 @test_encodekey128_u32(i32 %htype, <2 x i64> %key, ptr nocapture %h0, ; X64-NEXT: movaps %xmm2, (%rcx) ; X64-NEXT: retq ; -; X32-LABEL: test_encodekey128_u32: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: encodekey128 %eax, %eax -; X32-NEXT: vmovaps %xmm0, (%esi) -; X32-NEXT: vmovaps %xmm1, (%edx) -; X32-NEXT: vmovaps %xmm2, (%ecx) -; X32-NEXT: popl %esi -; X32-NEXT: retl +; X86-LABEL: test_encodekey128_u32: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: encodekey128 %eax, %eax +; X86-NEXT: vmovaps %xmm0, (%esi) +; X86-NEXT: vmovaps %xmm1, (%edx) +; X86-NEXT: vmovaps %xmm2, (%ecx) +; X86-NEXT: popl %esi +; X86-NEXT: retl entry: %0 = tail call { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey128(i32 %htype, <2 x i64> %key) %1 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %0, 1 @@ -77,23 +77,23 @@ define i32 @test_encodekey256_u32(i32 %htype, <2 x i64> %key_lo, <2 x i64> %key_ ; X64-NEXT: movaps %xmm3, (%r8) ; X64-NEXT: retq ; -; X32-LABEL: test_encodekey256_u32: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %edi -; X32-NEXT: pushl %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %edx -; X32-NEXT: movl {{[0-9]+}}(%esp), %esi -; X32-NEXT: movl {{[0-9]+}}(%esp), %edi -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: encodekey256 %eax, %eax -; X32-NEXT: vmovaps %xmm0, (%edi) -; X32-NEXT: vmovaps %xmm1, (%esi) -; X32-NEXT: vmovaps %xmm2, (%edx) -; X32-NEXT: vmovaps %xmm3, (%ecx) -; X32-NEXT: popl %esi -; X32-NEXT: popl %edi -; X32-NEXT: retl +; X86-LABEL: test_encodekey256_u32: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %edi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: encodekey256 %eax, %eax +; X86-NEXT: vmovaps %xmm0, (%edi) +; X86-NEXT: vmovaps %xmm1, (%esi) +; X86-NEXT: vmovaps %xmm2, (%edx) +; X86-NEXT: vmovaps %xmm3, (%ecx) +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl entry: %0 = tail call { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.encodekey256(i32 %htype, <2 x i64> %key_lo, <2 x i64> %key_hi) %1 = extractvalue { i32, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %0, 1 @@ -116,14 +116,14 @@ define i8 @test_mm_aesenc128kl_u8(<2 x i64> %data, ptr %h, ptr %out) { ; X64-NEXT: movaps %xmm0, (%rsi) ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesenc128kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: aesenc128kl (%eax), %xmm0 -; X32-NEXT: sete %al -; X32-NEXT: vmovaps %xmm0, (%ecx) -; X32-NEXT: retl +; X86-LABEL: test_mm_aesenc128kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: aesenc128kl (%eax), %xmm0 +; X86-NEXT: sete %al +; X86-NEXT: vmovaps %xmm0, (%ecx) +; X86-NEXT: retl entry: %0 = tail call { i8, <2 x i64> } @llvm.x86.aesenc128kl(<2 x i64> %data, ptr %h) %1 = extractvalue { i8, <2 x i64> } %0, 1 @@ -140,14 +140,14 @@ define i8 @test_mm_aesdec128kl_u8(<2 x i64> %data, ptr %h, ptr %out) { ; X64-NEXT: movaps %xmm0, (%rsi) ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesdec128kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: aesdec128kl (%eax), %xmm0 -; X32-NEXT: sete %al -; X32-NEXT: vmovaps %xmm0, (%ecx) -; X32-NEXT: retl +; X86-LABEL: test_mm_aesdec128kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: aesdec128kl (%eax), %xmm0 +; X86-NEXT: sete %al +; X86-NEXT: vmovaps %xmm0, (%ecx) +; X86-NEXT: retl entry: %0 = tail call { i8, <2 x i64> } @llvm.x86.aesdec128kl(<2 x i64> %data, ptr %h) %1 = extractvalue { i8, <2 x i64> } %0, 1 @@ -164,14 +164,14 @@ define i8 @test_mm_aesenc256kl_u8(<2 x i64> %data, ptr %h, ptr %out) { ; X64-NEXT: movaps %xmm0, (%rsi) ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesenc256kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: aesenc256kl (%eax), %xmm0 -; X32-NEXT: sete %al -; X32-NEXT: vmovaps %xmm0, (%ecx) -; X32-NEXT: retl +; X86-LABEL: test_mm_aesenc256kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: aesenc256kl (%eax), %xmm0 +; X86-NEXT: sete %al +; X86-NEXT: vmovaps %xmm0, (%ecx) +; X86-NEXT: retl entry: %0 = tail call { i8, <2 x i64> } @llvm.x86.aesenc256kl(<2 x i64> %data, ptr %h) %1 = extractvalue { i8, <2 x i64> } %0, 1 @@ -188,14 +188,14 @@ define i8 @test_mm_aesdec256kl_u8(<2 x i64> %data, ptr %h, ptr %out) { ; X64-NEXT: movaps %xmm0, (%rsi) ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesdec256kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: movl {{[0-9]+}}(%esp), %eax -; X32-NEXT: aesdec256kl (%eax), %xmm0 -; X32-NEXT: sete %al -; X32-NEXT: vmovaps %xmm0, (%ecx) -; X32-NEXT: retl +; X86-LABEL: test_mm_aesdec256kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: aesdec256kl (%eax), %xmm0 +; X86-NEXT: sete %al +; X86-NEXT: vmovaps %xmm0, (%ecx) +; X86-NEXT: retl entry: %0 = tail call { i8, <2 x i64> } @llvm.x86.aesdec256kl(<2 x i64> %data, ptr %h) %1 = extractvalue { i8, <2 x i64> } %0, 1 @@ -224,39 +224,39 @@ define i8 @test_mm_aesencwide128kl_u8(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x ; X64-NEXT: popq %rbx ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesencwide128kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %ebp -; X32-NEXT: movl %esp, %ebp -; X32-NEXT: andl $-16, %esp -; X32-NEXT: subl $16, %esp -; X32-NEXT: vmovaps 24(%ebp), %xmm3 -; X32-NEXT: vmovaps 40(%ebp), %xmm4 -; X32-NEXT: vmovaps 56(%ebp), %xmm5 -; X32-NEXT: vmovaps 72(%ebp), %xmm6 -; X32-NEXT: vmovaps 88(%ebp), %xmm7 -; X32-NEXT: movl 8(%ebp), %eax -; X32-NEXT: aesencwide128kl (%eax) -; X32-NEXT: movl 104(%ebp), %eax -; X32-NEXT: vmovaps %xmm0, (%eax) -; X32-NEXT: movl 108(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 112(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 116(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 120(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 124(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 128(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 132(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: sete %al -; X32-NEXT: movl %ebp, %esp -; X32-NEXT: popl %ebp -; X32-NEXT: retl +; X86-LABEL: test_mm_aesencwide128kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %ebp +; X86-NEXT: movl %esp, %ebp +; X86-NEXT: andl $-16, %esp +; X86-NEXT: subl $16, %esp +; X86-NEXT: vmovaps 24(%ebp), %xmm3 +; X86-NEXT: vmovaps 40(%ebp), %xmm4 +; X86-NEXT: vmovaps 56(%ebp), %xmm5 +; X86-NEXT: vmovaps 72(%ebp), %xmm6 +; X86-NEXT: vmovaps 88(%ebp), %xmm7 +; X86-NEXT: movl 8(%ebp), %eax +; X86-NEXT: aesencwide128kl (%eax) +; X86-NEXT: movl 104(%ebp), %eax +; X86-NEXT: vmovaps %xmm0, (%eax) +; X86-NEXT: movl 108(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 112(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 116(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 120(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 124(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 128(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 132(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: sete %al +; X86-NEXT: movl %ebp, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl entry: %0 = call { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.aesencwide128kl(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, <2 x i64> %v4, <2 x i64> %v5, <2 x i64> %v6, <2 x i64> %v7) %1 = extractvalue { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %0, 1 @@ -299,39 +299,39 @@ define i8 @test_mm_aesdecwide128kl_u8(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x ; X64-NEXT: popq %rbx ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesdecwide128kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %ebp -; X32-NEXT: movl %esp, %ebp -; X32-NEXT: andl $-16, %esp -; X32-NEXT: subl $16, %esp -; X32-NEXT: vmovaps 24(%ebp), %xmm3 -; X32-NEXT: vmovaps 40(%ebp), %xmm4 -; X32-NEXT: vmovaps 56(%ebp), %xmm5 -; X32-NEXT: vmovaps 72(%ebp), %xmm6 -; X32-NEXT: vmovaps 88(%ebp), %xmm7 -; X32-NEXT: movl 8(%ebp), %eax -; X32-NEXT: aesdecwide128kl (%eax) -; X32-NEXT: movl 104(%ebp), %eax -; X32-NEXT: vmovaps %xmm0, (%eax) -; X32-NEXT: movl 108(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 112(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 116(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 120(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 124(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 128(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 132(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: sete %al -; X32-NEXT: movl %ebp, %esp -; X32-NEXT: popl %ebp -; X32-NEXT: retl +; X86-LABEL: test_mm_aesdecwide128kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %ebp +; X86-NEXT: movl %esp, %ebp +; X86-NEXT: andl $-16, %esp +; X86-NEXT: subl $16, %esp +; X86-NEXT: vmovaps 24(%ebp), %xmm3 +; X86-NEXT: vmovaps 40(%ebp), %xmm4 +; X86-NEXT: vmovaps 56(%ebp), %xmm5 +; X86-NEXT: vmovaps 72(%ebp), %xmm6 +; X86-NEXT: vmovaps 88(%ebp), %xmm7 +; X86-NEXT: movl 8(%ebp), %eax +; X86-NEXT: aesdecwide128kl (%eax) +; X86-NEXT: movl 104(%ebp), %eax +; X86-NEXT: vmovaps %xmm0, (%eax) +; X86-NEXT: movl 108(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 112(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 116(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 120(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 124(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 128(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 132(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: sete %al +; X86-NEXT: movl %ebp, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl entry: %0 = call { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.aesdecwide128kl(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, <2 x i64> %v4, <2 x i64> %v5, <2 x i64> %v6, <2 x i64> %v7) %1 = extractvalue { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %0, 1 @@ -374,39 +374,39 @@ define i8 @test_mm_aesencwide256kl_u8(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x ; X64-NEXT: popq %rbx ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesencwide256kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %ebp -; X32-NEXT: movl %esp, %ebp -; X32-NEXT: andl $-16, %esp -; X32-NEXT: subl $16, %esp -; X32-NEXT: vmovaps 24(%ebp), %xmm3 -; X32-NEXT: vmovaps 40(%ebp), %xmm4 -; X32-NEXT: vmovaps 56(%ebp), %xmm5 -; X32-NEXT: vmovaps 72(%ebp), %xmm6 -; X32-NEXT: vmovaps 88(%ebp), %xmm7 -; X32-NEXT: movl 8(%ebp), %eax -; X32-NEXT: aesencwide256kl (%eax) -; X32-NEXT: movl 104(%ebp), %eax -; X32-NEXT: vmovaps %xmm0, (%eax) -; X32-NEXT: movl 108(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 112(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 116(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 120(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 124(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 128(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 132(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: sete %al -; X32-NEXT: movl %ebp, %esp -; X32-NEXT: popl %ebp -; X32-NEXT: retl +; X86-LABEL: test_mm_aesencwide256kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %ebp +; X86-NEXT: movl %esp, %ebp +; X86-NEXT: andl $-16, %esp +; X86-NEXT: subl $16, %esp +; X86-NEXT: vmovaps 24(%ebp), %xmm3 +; X86-NEXT: vmovaps 40(%ebp), %xmm4 +; X86-NEXT: vmovaps 56(%ebp), %xmm5 +; X86-NEXT: vmovaps 72(%ebp), %xmm6 +; X86-NEXT: vmovaps 88(%ebp), %xmm7 +; X86-NEXT: movl 8(%ebp), %eax +; X86-NEXT: aesencwide256kl (%eax) +; X86-NEXT: movl 104(%ebp), %eax +; X86-NEXT: vmovaps %xmm0, (%eax) +; X86-NEXT: movl 108(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 112(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 116(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 120(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 124(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 128(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 132(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: sete %al +; X86-NEXT: movl %ebp, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl entry: %0 = call { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.aesencwide256kl(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, <2 x i64> %v4, <2 x i64> %v5, <2 x i64> %v6, <2 x i64> %v7) %1 = extractvalue { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %0, 1 @@ -449,39 +449,39 @@ define i8 @test_mm_aesdecwide256kl_u8(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x ; X64-NEXT: popq %rbx ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesdecwide256kl_u8: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %ebp -; X32-NEXT: movl %esp, %ebp -; X32-NEXT: andl $-16, %esp -; X32-NEXT: subl $16, %esp -; X32-NEXT: vmovaps 24(%ebp), %xmm3 -; X32-NEXT: vmovaps 40(%ebp), %xmm4 -; X32-NEXT: vmovaps 56(%ebp), %xmm5 -; X32-NEXT: vmovaps 72(%ebp), %xmm6 -; X32-NEXT: vmovaps 88(%ebp), %xmm7 -; X32-NEXT: movl 8(%ebp), %eax -; X32-NEXT: aesdecwide256kl (%eax) -; X32-NEXT: movl 104(%ebp), %eax -; X32-NEXT: vmovaps %xmm0, (%eax) -; X32-NEXT: movl 108(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 112(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 116(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 120(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 124(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 128(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 132(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: sete %al -; X32-NEXT: movl %ebp, %esp -; X32-NEXT: popl %ebp -; X32-NEXT: retl +; X86-LABEL: test_mm_aesdecwide256kl_u8: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %ebp +; X86-NEXT: movl %esp, %ebp +; X86-NEXT: andl $-16, %esp +; X86-NEXT: subl $16, %esp +; X86-NEXT: vmovaps 24(%ebp), %xmm3 +; X86-NEXT: vmovaps 40(%ebp), %xmm4 +; X86-NEXT: vmovaps 56(%ebp), %xmm5 +; X86-NEXT: vmovaps 72(%ebp), %xmm6 +; X86-NEXT: vmovaps 88(%ebp), %xmm7 +; X86-NEXT: movl 8(%ebp), %eax +; X86-NEXT: aesdecwide256kl (%eax) +; X86-NEXT: movl 104(%ebp), %eax +; X86-NEXT: vmovaps %xmm0, (%eax) +; X86-NEXT: movl 108(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 112(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 116(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 120(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 124(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 128(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 132(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: sete %al +; X86-NEXT: movl %ebp, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl entry: %0 = call { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.aesdecwide256kl(ptr %p, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, <2 x i64> %v4, <2 x i64> %v5, <2 x i64> %v6, <2 x i64> %v7) %1 = extractvalue { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %0, 1 @@ -516,13 +516,13 @@ define i8 @test_mm_aesenc256kl_u8_global(<2 x i64> %data, ptr %out) { ; X64-NEXT: movaps %xmm0, (%rdi) ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesenc256kl_u8_global: -; X32: # %bb.0: # %entry -; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X32-NEXT: aesenc256kl foo, %xmm0 -; X32-NEXT: sete %al -; X32-NEXT: vmovaps %xmm0, (%ecx) -; X32-NEXT: retl +; X86-LABEL: test_mm_aesenc256kl_u8_global: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: aesenc256kl foo, %xmm0 +; X86-NEXT: sete %al +; X86-NEXT: vmovaps %xmm0, (%ecx) +; X86-NEXT: retl entry: %0 = tail call { i8, <2 x i64> } @llvm.x86.aesenc256kl(<2 x i64> %data, ptr @foo) %1 = extractvalue { i8, <2 x i64> } %0, 1 @@ -548,38 +548,38 @@ define i8 @test_mm_aesdecwide256kl_u8_global(<2 x i64> %v0, <2 x i64> %v1, <2 x ; X64-NEXT: movaps %xmm1, (%r10) ; X64-NEXT: retq ; -; X32-LABEL: test_mm_aesdecwide256kl_u8_global: -; X32: # %bb.0: # %entry -; X32-NEXT: pushl %ebp -; X32-NEXT: movl %esp, %ebp -; X32-NEXT: andl $-16, %esp -; X32-NEXT: subl $16, %esp -; X32-NEXT: movl 88(%ebp), %eax -; X32-NEXT: vmovaps 8(%ebp), %xmm3 -; X32-NEXT: vmovaps 24(%ebp), %xmm4 -; X32-NEXT: vmovaps 40(%ebp), %xmm5 -; X32-NEXT: vmovaps 56(%ebp), %xmm6 -; X32-NEXT: vmovaps 72(%ebp), %xmm7 -; X32-NEXT: aesdecwide256kl foo -; X32-NEXT: vmovaps %xmm0, (%eax) -; X32-NEXT: movl 92(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 96(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 100(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 104(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 108(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 112(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: movl 116(%ebp), %eax -; X32-NEXT: vmovaps %xmm1, (%eax) -; X32-NEXT: sete %al -; X32-NEXT: movl %ebp, %esp -; X32-NEXT: popl %ebp -; X32-NEXT: retl +; X86-LABEL: test_mm_aesdecwide256kl_u8_global: +; X86: # %bb.0: # %entry +; X86-NEXT: pushl %ebp +; X86-NEXT: movl %esp, %ebp +; X86-NEXT: andl $-16, %esp +; X86-NEXT: subl $16, %esp +; X86-NEXT: movl 88(%ebp), %eax +; X86-NEXT: vmovaps 8(%ebp), %xmm3 +; X86-NEXT: vmovaps 24(%ebp), %xmm4 +; X86-NEXT: vmovaps 40(%ebp), %xmm5 +; X86-NEXT: vmovaps 56(%ebp), %xmm6 +; X86-NEXT: vmovaps 72(%ebp), %xmm7 +; X86-NEXT: aesdecwide256kl foo +; X86-NEXT: vmovaps %xmm0, (%eax) +; X86-NEXT: movl 92(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 96(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 100(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 104(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 108(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 112(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: movl 116(%ebp), %eax +; X86-NEXT: vmovaps %xmm1, (%eax) +; X86-NEXT: sete %al +; X86-NEXT: movl %ebp, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl entry: %0 = call { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.x86.aesdecwide256kl(ptr @foo, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, <2 x i64> %v4, <2 x i64> %v5, <2 x i64> %v6, <2 x i64> %v7) %1 = extractvalue { i8, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } %0, 1 -- GitLab From 06f1e10908e624c1e90a0c647e9f74826ad3f011 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Fri, 5 Jan 2024 14:41:44 +0100 Subject: [PATCH 114/728] [mlir][nvvm] Add clock and clock64 special registers (#77088) Tihs PR adds `clock` and `clock64` special registers to NVVM dialect. --- mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 5 +++++ mlir/test/Target/LLVMIR/nvvmir.mlir | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td index 57986f291de7..52857164ffaf 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td @@ -156,6 +156,11 @@ def NVVM_GridInClusterDimZOp : NVVM_SpecialRegisterOp<"read.ptx.sreg.cluster.nct def NVVM_ClusterId : NVVM_SpecialRegisterOp<"read.ptx.sreg.cluster.ctarank">; def NVVM_ClusterDim : NVVM_SpecialRegisterOp<"read.ptx.sreg.cluster.nctarank">; +//===----------------------------------------------------------------------===// +// Clock registers +def NVVM_ClockOp : NVVM_SpecialRegisterOp<"read.ptx.sreg.clock">; +def NVVM_Clock64Op : NVVM_SpecialRegisterOp<"read.ptx.sreg.clock64">; + //===----------------------------------------------------------------------===// // NVVM approximate op definitions //===----------------------------------------------------------------------===// diff --git a/mlir/test/Target/LLVMIR/nvvmir.mlir b/mlir/test/Target/LLVMIR/nvvmir.mlir index 3fed2c24b314..6076fce598fb 100644 --- a/mlir/test/Target/LLVMIR/nvvmir.mlir +++ b/mlir/test/Target/LLVMIR/nvvmir.mlir @@ -58,6 +58,10 @@ llvm.func @nvvm_special_regs() -> i32 { %27 = nvvm.read.ptx.sreg.cluster.ctarank : i32 // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.cluster.nctarank %28 = nvvm.read.ptx.sreg.cluster.nctarank : i32 + // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.clock + %29 = nvvm.read.ptx.sreg.clock : i32 + // CHECK: call i64 @llvm.nvvm.read.ptx.sreg.clock64 + %30 = nvvm.read.ptx.sreg.clock64 : i64 llvm.return %1 : i32 } -- GitLab From f07aba4bc1fbd8301b09e2114ebc4149d2439cac Mon Sep 17 00:00:00 2001 From: Phoebe Wang Date: Fri, 5 Jan 2024 21:53:47 +0800 Subject: [PATCH 115/728] [X86] Add ABI handling for __float128 to match with GCC (#75156) Fixes #74601 --- clang/docs/ReleaseNotes.rst | 1 + clang/lib/CodeGen/Targets/X86.cpp | 3 +++ clang/test/CodeGen/X86/fp128-abi.c | 36 ++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 clang/test/CodeGen/X86/fp128-abi.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 3c63ad96ca2e..f5159cdc8a3b 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -926,6 +926,7 @@ X86 Support * Support intrinsic of ``_uwrmsr``. - Support ISA of ``AVX10.1``. - ``-march=pantherlake`` and ``-march=clearwaterforest`` are now supported. +- Added ABI handling for ``__float128`` to match with GCC. Arm and AArch64 Support ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp index 2af240350438..d053f41ab168 100644 --- a/clang/lib/CodeGen/Targets/X86.cpp +++ b/clang/lib/CodeGen/Targets/X86.cpp @@ -1797,6 +1797,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo, } else if (k == BuiltinType::Float || k == BuiltinType::Double || k == BuiltinType::Float16 || k == BuiltinType::BFloat16) { Current = SSE; + } else if (k == BuiltinType::Float128) { + Lo = SSE; + Hi = SSEUp; } else if (k == BuiltinType::LongDouble) { const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat(); if (LDF == &llvm::APFloat::IEEEquad()) { diff --git a/clang/test/CodeGen/X86/fp128-abi.c b/clang/test/CodeGen/X86/fp128-abi.c new file mode 100644 index 000000000000..2a0ae5009338 --- /dev/null +++ b/clang/test/CodeGen/X86/fp128-abi.c @@ -0,0 +1,36 @@ +// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -target-feature +sse2 < %s | FileCheck %s --check-prefixes=CHECK +// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -target-feature -sse2 < %s | FileCheck %s --check-prefixes=CHECK + +struct st1 { + __float128 a; +}; + +struct st1 h1(__float128 a) { + // CHECK: define{{.*}}fp128 @h1(fp128 + struct st1 x; + x.a = a; + return x; +} + +__float128 h2(struct st1 x) { + // CHECK: define{{.*}}fp128 @h2(fp128 + return x.a; +} + +struct st2 { + __float128 a; + int b; +}; + +struct st2 h3(__float128 a, int b) { + // CHECK: define{{.*}}void @h3(ptr {{.*}}sret(%struct.st2) + struct st2 x; + x.a = a; + x.b = b; + return x; +} + +__float128 h4(struct st2 x) { + // CHECK: define{{.*}}fp128 @h4(ptr {{.*}}byval(%struct.st2) + return x.a; +} -- GitLab From 2b88bd110cbe61e1e3ef764d0362a75dc7c9cd50 Mon Sep 17 00:00:00 2001 From: Felipe de Azevedo Piovezan Date: Fri, 5 Jan 2024 11:01:19 -0300 Subject: [PATCH 116/728] [AsmPrinter][Dwarf5][nfc] Remove template from AccelTable class (#76296) This template is no longer used. --- llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp | 62 +++++++++------------- 1 file changed, 25 insertions(+), 37 deletions(-) diff --git a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp index 30ea7eef3a12..bf580269eca6 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp @@ -182,11 +182,7 @@ public: /// Class responsible for emitting a DWARF v5 Accelerator Table. The only /// public function is emit(), which performs the actual emission. /// -/// The class is templated in its data type. This allows us to emit both dyamic -/// and static data entries. A callback abstract the logic to provide a CU -/// index for a given entry, which is different per data type, but identical -/// for every entry in the same table. -template +/// A callback abstracts the logic to provide a CU index for a given entry. class Dwarf5AccelTableWriter : public AccelTableWriter { struct Header { uint16_t Version = 5; @@ -216,7 +212,7 @@ class Dwarf5AccelTableWriter : public AccelTableWriter { ArrayRef> CompUnits; ArrayRef> TypeUnits; llvm::function_ref( - const DataT &)> + const DWARF5AccelTableData &)> getIndexForEntry; MCSymbol *ContributionEnd = nullptr; MCSymbol *AbbrevStart = Asm->createTempSymbol("names_abbrev_start"); @@ -232,7 +228,7 @@ class Dwarf5AccelTableWriter : public AccelTableWriter { void emitBuckets() const; void emitStringOffsets() const; void emitAbbrevs() const; - void emitEntry(const DataT &Entry) const; + void emitEntry(const DWARF5AccelTableData &Entry) const; void emitData() const; public: @@ -240,8 +236,8 @@ public: AsmPrinter *Asm, const AccelTableBase &Contents, ArrayRef> CompUnits, ArrayRef> TypeUnits, - llvm::function_ref< - std::optional(const DataT &)> + llvm::function_ref( + const DWARF5AccelTableData &)> getIndexForEntry, bool IsSplitDwarf); @@ -370,8 +366,7 @@ DWARF5AccelTableData::DWARF5AccelTableData(const DIE &Die, const bool IsTU) : OffsetVal(&Die), DieTag(Die.getTag()), UnitID(UnitID), IsTU(IsTU) {} -template -void Dwarf5AccelTableWriter::Header::emit(Dwarf5AccelTableWriter &Ctx) { +void Dwarf5AccelTableWriter::Header::emit(Dwarf5AccelTableWriter &Ctx) { assert(CompUnitCount > 0 && "Index must have at least one CU."); AsmPrinter *Asm = Ctx.Asm; @@ -417,14 +412,14 @@ static uint32_t constructAbbreviationTag( AbbrvTag |= Tag << LowerBitSize; return AbbrvTag; } -template -void Dwarf5AccelTableWriter::populateAbbrevsMap() { +void Dwarf5AccelTableWriter::populateAbbrevsMap() { for (auto &Bucket : Contents.getBuckets()) { for (auto *Hash : Bucket) { for (auto *Value : Hash->Values) { std::optional EntryRet = - getIndexForEntry(*static_cast(Value)); - unsigned Tag = static_cast(Value)->getDieTag(); + getIndexForEntry(*static_cast(Value)); + unsigned Tag = + static_cast(Value)->getDieTag(); uint32_t AbbrvTag = constructAbbreviationTag(Tag, EntryRet); if (Abbreviations.count(AbbrvTag) == 0) { SmallVector UA; @@ -438,8 +433,7 @@ void Dwarf5AccelTableWriter::populateAbbrevsMap() { } } -template -void Dwarf5AccelTableWriter::emitCUList() const { +void Dwarf5AccelTableWriter::emitCUList() const { for (const auto &CU : enumerate(CompUnits)) { Asm->OutStreamer->AddComment("Compilation unit " + Twine(CU.index())); if (std::holds_alternative(CU.value())) @@ -449,8 +443,7 @@ void Dwarf5AccelTableWriter::emitCUList() const { } } -template -void Dwarf5AccelTableWriter::emitTUList() const { +void Dwarf5AccelTableWriter::emitTUList() const { for (const auto &TU : enumerate(TypeUnits)) { Asm->OutStreamer->AddComment("Type unit " + Twine(TU.index())); if (std::holds_alternative(TU.value())) @@ -462,8 +455,7 @@ void Dwarf5AccelTableWriter::emitTUList() const { } } -template -void Dwarf5AccelTableWriter::emitBuckets() const { +void Dwarf5AccelTableWriter::emitBuckets() const { uint32_t Index = 1; for (const auto &Bucket : enumerate(Contents.getBuckets())) { Asm->OutStreamer->AddComment("Bucket " + Twine(Bucket.index())); @@ -472,8 +464,7 @@ void Dwarf5AccelTableWriter::emitBuckets() const { } } -template -void Dwarf5AccelTableWriter::emitStringOffsets() const { +void Dwarf5AccelTableWriter::emitStringOffsets() const { for (const auto &Bucket : enumerate(Contents.getBuckets())) { for (auto *Hash : Bucket.value()) { DwarfStringPoolEntryRef String = Hash->Name; @@ -484,8 +475,7 @@ void Dwarf5AccelTableWriter::emitStringOffsets() const { } } -template -void Dwarf5AccelTableWriter::emitAbbrevs() const { +void Dwarf5AccelTableWriter::emitAbbrevs() const { Asm->OutStreamer->emitLabel(AbbrevStart); for (const auto &Abbrev : Abbreviations) { Asm->OutStreamer->AddComment("Abbrev code"); @@ -506,8 +496,8 @@ void Dwarf5AccelTableWriter::emitAbbrevs() const { Asm->OutStreamer->emitLabel(AbbrevEnd); } -template -void Dwarf5AccelTableWriter::emitEntry(const DataT &Entry) const { +void Dwarf5AccelTableWriter::emitEntry( + const DWARF5AccelTableData &Entry) const { std::optional EntryRet = getIndexForEntry(Entry); uint32_t AbbrvTag = constructAbbreviationTag(Entry.getDieTag(), EntryRet); @@ -537,27 +527,26 @@ void Dwarf5AccelTableWriter::emitEntry(const DataT &Entry) const { } } -template void Dwarf5AccelTableWriter::emitData() const { +void Dwarf5AccelTableWriter::emitData() const { Asm->OutStreamer->emitLabel(EntryPool); for (auto &Bucket : Contents.getBuckets()) { for (auto *Hash : Bucket) { // Remember to emit the label for our offset. Asm->OutStreamer->emitLabel(Hash->Sym); for (const auto *Value : Hash->Values) - emitEntry(*static_cast(Value)); + emitEntry(*static_cast(Value)); Asm->OutStreamer->AddComment("End of list: " + Hash->Name.getString()); Asm->emitInt8(0); } } } -template -Dwarf5AccelTableWriter::Dwarf5AccelTableWriter( +Dwarf5AccelTableWriter::Dwarf5AccelTableWriter( AsmPrinter *Asm, const AccelTableBase &Contents, ArrayRef> CompUnits, ArrayRef> TypeUnits, - llvm::function_ref< - std::optional(const DataT &)> + llvm::function_ref( + const DWARF5AccelTableData &)> getIndexForEntry, bool IsSplitDwarf) : AccelTableWriter(Asm, Contents, false), @@ -570,7 +559,7 @@ Dwarf5AccelTableWriter::Dwarf5AccelTableWriter( populateAbbrevsMap(); } -template void Dwarf5AccelTableWriter::emit() { +void Dwarf5AccelTableWriter::emit() { Header.emit(*this); emitCUList(); emitTUList(); @@ -635,7 +624,7 @@ void llvm::emitDWARF5AccelTable( DIEInteger::BestForm(/*IsSigned*/ false, CompUnits.size() - 1); dwarf::Form TUIndexForm = DIEInteger::BestForm(/*IsSigned*/ false, TypeUnits.size() - 1); - Dwarf5AccelTableWriter( + Dwarf5AccelTableWriter( Asm, Contents, CompUnits, TypeUnits, [&](const DWARF5AccelTableData &Entry) -> std::optional { @@ -667,8 +656,7 @@ void llvm::emitDWARF5AccelTable( getIndexForEntry) { std::vector> TypeUnits; Contents.finalize(Asm, "names"); - Dwarf5AccelTableWriter(Asm, Contents, CUs, TypeUnits, - getIndexForEntry, false) + Dwarf5AccelTableWriter(Asm, Contents, CUs, TypeUnits, getIndexForEntry, false) .emit(); } -- GitLab From a776740d6296520b8bde156aa3f8d9ecb32cddd9 Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Fri, 5 Jan 2024 14:08:53 +0000 Subject: [PATCH 117/728] [DebugInfo] Correctly track metadata slots for DPValues (#76941) Currently, the AsmWriter can print DPValues, but does not consider them when creating slots for metadata, which can result in erroneous output where metadata is numbered incorrectly. This patch modifies the ModuleSlotTracker to correctly track slots for metadata that appears in DPValues. --- llvm/lib/IR/AsmWriter.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 95cdec722062..278cdfce4110 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -859,6 +859,9 @@ private: /// Add all of the metadata from an instruction. void processInstructionMetadata(const Instruction &I); + + /// Add all of the metadata from an instruction. + void processDPValueMetadata(const DPValue &DPV); }; } // end namespace llvm @@ -1126,11 +1129,19 @@ void SlotTracker::processGlobalObjectMetadata(const GlobalObject &GO) { void SlotTracker::processFunctionMetadata(const Function &F) { processGlobalObjectMetadata(F); for (auto &BB : F) { - for (auto &I : BB) + for (auto &I : BB) { + for (const DPValue &DPV : I.getDbgValueRange()) + processDPValueMetadata(DPV); processInstructionMetadata(I); + } } } +void SlotTracker::processDPValueMetadata(const DPValue &DPV) { + CreateMetadataSlot(DPV.getVariable()); + CreateMetadataSlot(DPV.getDebugLoc()); +} + void SlotTracker::processInstructionMetadata(const Instruction &I) { // Process metadata used directly by intrinsics. if (const CallInst *CI = dyn_cast(&I)) -- GitLab From 65df69619c49717da64140baddda0f04ed62ffdf Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 5 Jan 2024 15:28:36 +0100 Subject: [PATCH 118/728] [ConstraintElim] Add tests for signed induction variables (NFC) --- .../monotonic-int-phis-signed.ll | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll diff --git a/llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll b/llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll new file mode 100644 index 000000000000..1e95fab5c10c --- /dev/null +++ b/llvm/test/Transforms/ConstraintElimination/monotonic-int-phis-signed.ll @@ -0,0 +1,297 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=constraint-elimination < %s | FileCheck %s + +declare void @use(i1) + +define void @signed_iv_step_1(i64 %end) { +; CHECK-LABEL: define void @signed_iv_step_1( +; CHECK-SAME: i64 [[END:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[PRECOND:%.*]] = icmp sge i64 [[END]], -10 +; CHECK-NEXT: br i1 [[PRECOND]], label [[LOOP:%.*]], label [[EXIT:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ -10, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV]], [[END]] +; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i64 [[IV]], [[END]] +; CHECK-NEXT: call void @use(i1 [[CMP2]]) +; CHECK-NEXT: [[CMP3:%.*]] = icmp sge i64 [[IV]], -10 +; CHECK-NEXT: call void @use(i1 [[CMP3]]) +; CHECK-NEXT: br label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %precond = icmp sge i64 %end, -10 + br i1 %precond, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv.next, %loop.latch ], [ -10, %entry ] + %iv.next = add i64 %iv, 1 + %cmp.i.not = icmp eq i64 %iv, %end + br i1 %cmp.i.not, label %exit, label %loop.latch + +loop.latch: + %cmp2 = icmp slt i64 %iv, %end + call void @use(i1 %cmp2) + %cmp3 = icmp sge i64 %iv, -10 + call void @use(i1 %cmp3) + br label %loop + +exit: + ret void +} + +define void @signed_iv_step_4(i64 %count) { +; CHECK-LABEL: define void @signed_iv_step_4( +; CHECK-SAME: i64 [[COUNT:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[END:%.*]] = shl nsw i64 [[COUNT]], 2 +; CHECK-NEXT: [[PRECOND:%.*]] = icmp sgt i64 [[COUNT]], -1 +; CHECK-NEXT: br i1 [[PRECOND]], label [[LOOP:%.*]], label [[EXIT:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4 +; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV]], [[END]] +; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: call void @use(i1 true) +; CHECK-NEXT: call void @use(i1 true) +; CHECK-NEXT: br label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %end = shl nsw i64 %count, 2 + %precond = icmp sgt i64 %count, -1 + br i1 %precond, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ] + %iv.next = add i64 %iv, 4 + %cmp.i.not = icmp eq i64 %iv, %end + br i1 %cmp.i.not, label %exit, label %loop.latch + +loop.latch: + %cmp2 = icmp slt i64 %iv, %end + call void @use(i1 %cmp2) + %cmp3 = icmp sge i64 %iv, 0 + call void @use(i1 %cmp3) + br label %loop + +exit: + ret void +} + +define void @signed_iv_step_4_missing_precond(i64 %count) { +; CHECK-LABEL: define void @signed_iv_step_4_missing_precond( +; CHECK-SAME: i64 [[COUNT:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[END:%.*]] = shl i64 [[COUNT]], 2 +; CHECK-NEXT: [[PRECOND:%.*]] = icmp sgt i64 [[COUNT]], -1 +; CHECK-NEXT: br i1 [[PRECOND]], label [[LOOP:%.*]], label [[EXIT:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4 +; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV]], [[END]] +; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i64 [[IV]], [[END]] +; CHECK-NEXT: call void @use(i1 [[CMP2]]) +; CHECK-NEXT: [[CMP3:%.*]] = icmp sge i64 [[IV]], 0 +; CHECK-NEXT: call void @use(i1 [[CMP3]]) +; CHECK-NEXT: br label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %end = shl i64 %count, 2 + %precond = icmp sgt i64 %count, -1 + br i1 %precond, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ] + %iv.next = add i64 %iv, 4 + %cmp.i.not = icmp eq i64 %iv, %end + br i1 %cmp.i.not, label %exit, label %loop.latch + +loop.latch: + %cmp2 = icmp slt i64 %iv, %end + call void @use(i1 %cmp2) + %cmp3 = icmp sge i64 %iv, 0 + call void @use(i1 %cmp3) + br label %loop + +exit: + ret void +} + +define void @signed_iv_step_4_start_4(i64 %count) { +; CHECK-LABEL: define void @signed_iv_step_4_start_4( +; CHECK-SAME: i64 [[COUNT:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[END:%.*]] = shl nsw i64 [[COUNT]], 2 +; CHECK-NEXT: [[PRECOND:%.*]] = icmp sgt i64 [[COUNT]], 0 +; CHECK-NEXT: br i1 [[PRECOND]], label [[LOOP:%.*]], label [[EXIT:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 4, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4 +; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV]], [[END]] +; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i64 [[IV]], [[END]] +; CHECK-NEXT: call void @use(i1 [[CMP2]]) +; CHECK-NEXT: [[CMP3:%.*]] = icmp sge i64 [[IV]], 4 +; CHECK-NEXT: call void @use(i1 [[CMP3]]) +; CHECK-NEXT: br label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %end = shl nsw i64 %count, 2 + %precond = icmp sgt i64 %count, 0 + br i1 %precond, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv.next, %loop.latch ], [ 4, %entry ] + %iv.next = add i64 %iv, 4 + %cmp.i.not = icmp eq i64 %iv, %end + br i1 %cmp.i.not, label %exit, label %loop.latch + +loop.latch: + %cmp2 = icmp slt i64 %iv, %end + call void @use(i1 %cmp2) + %cmp3 = icmp sge i64 %iv, 4 + call void @use(i1 %cmp3) + br label %loop + +exit: + ret void +} + +define void @signed_iv_step_4_start_4_missing_precond(i64 %count) { +; CHECK-LABEL: define void @signed_iv_step_4_start_4_missing_precond( +; CHECK-SAME: i64 [[COUNT:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[END:%.*]] = shl nsw i64 [[COUNT]], 2 +; CHECK-NEXT: [[PRECOND:%.*]] = icmp sgt i64 [[COUNT]], -1 +; CHECK-NEXT: br i1 [[PRECOND]], label [[LOOP:%.*]], label [[EXIT:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 4, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4 +; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV]], [[END]] +; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i64 [[IV]], [[END]] +; CHECK-NEXT: call void @use(i1 [[CMP2]]) +; CHECK-NEXT: [[CMP3:%.*]] = icmp sge i64 [[IV]], 4 +; CHECK-NEXT: call void @use(i1 [[CMP3]]) +; CHECK-NEXT: br label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %end = shl nsw i64 %count, 2 + %precond = icmp sgt i64 %count, -1 + br i1 %precond, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv.next, %loop.latch ], [ 4, %entry ] + %iv.next = add i64 %iv, 4 + %cmp.i.not = icmp eq i64 %iv, %end + br i1 %cmp.i.not, label %exit, label %loop.latch + +loop.latch: + %cmp2 = icmp slt i64 %iv, %end + call void @use(i1 %cmp2) + %cmp3 = icmp sge i64 %iv, 4 + call void @use(i1 %cmp3) + br label %loop + +exit: + ret void +} + +define void @signed_iv_step_minus1(i64 %end) { +; CHECK-LABEL: define void @signed_iv_step_minus1( +; CHECK-SAME: i64 [[END:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[PRECOND:%.*]] = icmp sle i64 [[END]], 10 +; CHECK-NEXT: br i1 [[PRECOND]], label [[LOOP:%.*]], label [[EXIT:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 10, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1 +; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV]], [[END]] +; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i64 [[IV]], [[END]] +; CHECK-NEXT: call void @use(i1 [[CMP2]]) +; CHECK-NEXT: [[CMP3:%.*]] = icmp sle i64 [[IV]], 10 +; CHECK-NEXT: call void @use(i1 [[CMP3]]) +; CHECK-NEXT: br label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %precond = icmp sle i64 %end, 10 + br i1 %precond, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv.next, %loop.latch ], [ 10, %entry ] + %iv.next = add i64 %iv, -1 + %cmp.i.not = icmp eq i64 %iv, %end + br i1 %cmp.i.not, label %exit, label %loop.latch + +loop.latch: + %cmp2 = icmp sgt i64 %iv, %end + call void @use(i1 %cmp2) + %cmp3 = icmp sle i64 %iv, 10 + call void @use(i1 %cmp3) + br label %loop + +exit: + ret void +} + +define void @signed_iv_step_minus1_missing_precond(i64 %end) { +; CHECK-LABEL: define void @signed_iv_step_minus1_missing_precond( +; CHECK-SAME: i64 [[END:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[PRECOND:%.*]] = icmp sle i64 [[END]], 11 +; CHECK-NEXT: br i1 [[PRECOND]], label [[LOOP:%.*]], label [[EXIT:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 10, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1 +; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq i64 [[IV]], [[END]] +; CHECK-NEXT: br i1 [[CMP_I_NOT]], label [[EXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i64 [[IV]], [[END]] +; CHECK-NEXT: call void @use(i1 [[CMP2]]) +; CHECK-NEXT: [[CMP3:%.*]] = icmp sle i64 [[IV]], 10 +; CHECK-NEXT: call void @use(i1 [[CMP3]]) +; CHECK-NEXT: br label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %precond = icmp sle i64 %end, 11 + br i1 %precond, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv.next, %loop.latch ], [ 10, %entry ] + %iv.next = add i64 %iv, -1 + %cmp.i.not = icmp eq i64 %iv, %end + br i1 %cmp.i.not, label %exit, label %loop.latch + +loop.latch: + %cmp2 = icmp sgt i64 %iv, %end + call void @use(i1 %cmp2) + %cmp3 = icmp sle i64 %iv, 10 + call void @use(i1 %cmp3) + br label %loop + +exit: + ret void +} -- GitLab From a0e6b7c0429204ac42095be09bd1d5dcad4a052a Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 5 Jan 2024 22:44:04 +0800 Subject: [PATCH 119/728] [TableGen] Add a backend to generate MacroFusion predicators (#72222) `FusionPredicate` is used to predicate if target instruction matches the requirement. The targets can be firstMI, secondMI or both. The `Fusion` contains a list of `FusionPredicate`. The generated code will be like: ``` bool isNAME(const TargetInstrInfo &TII, const TargetSubtargetInfo &STI, const MachineInstr *FirstMI, const MachineInstr &SecondMI) { auto &MRI = SecondMI.getMF()->getRegInfo(); /* Predicates */ return true; } ``` A boilerplate class called `SimpleFusion` is added. `SimpleFusion` has a predefined structure of predicates and accepts predicate for `firstMI`, predicate for `secondMI` and epilog/prolog as arguments. The generated code for `SimpleFusion` will be like: ``` bool isNAME(const TargetInstrInfo &TII, const TargetSubtargetInfo &STI, const MachineInstr *FirstMI, const MachineInstr &SecondMI) { auto &MRI = SecondMI.getMF()->getRegInfo(); /* Prolog */ /* Predicate for `SecondMI` */ /* Wildcard */ /* Predicate for `FirstMI` */ /* Check One Use */ /* Tie registers */ /* Epilog */ return true; } ``` --- .../llvm/Target/TargetInstrPredicate.td | 6 + llvm/include/llvm/Target/TargetSchedule.td | 113 +++++++++ llvm/test/TableGen/MacroFusion.td | 97 +++++++ llvm/utils/TableGen/CMakeLists.txt | 1 + .../TableGen/MacroFusionPredicatorEmitter.cpp | 236 ++++++++++++++++++ llvm/utils/TableGen/PredicateExpander.cpp | 8 + llvm/utils/TableGen/PredicateExpander.h | 1 + 7 files changed, 462 insertions(+) create mode 100644 llvm/test/TableGen/MacroFusion.td create mode 100644 llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp diff --git a/llvm/include/llvm/Target/TargetInstrPredicate.td b/llvm/include/llvm/Target/TargetInstrPredicate.td index 9f2cde9d9230..82c4c7b23a49 100644 --- a/llvm/include/llvm/Target/TargetInstrPredicate.td +++ b/llvm/include/llvm/Target/TargetInstrPredicate.td @@ -95,6 +95,12 @@ class MCOperandPredicate : MCInstPredicate { // Return true if machine operand at position `Index` is a register operand. class CheckIsRegOperand : MCOperandPredicate; +// Return true if machine operand at position `Index` is a virtual register operand. +class CheckIsVRegOperand : MCOperandPredicate; + +// Return true if machine operand at position `Index` is not a virtual register operand. +class CheckIsNotVRegOperand : CheckNot>; + // Return true if machine operand at position `Index` is an immediate operand. class CheckIsImmOperand : MCOperandPredicate; diff --git a/llvm/include/llvm/Target/TargetSchedule.td b/llvm/include/llvm/Target/TargetSchedule.td index 949baa5d2105..2016d452afb6 100644 --- a/llvm/include/llvm/Target/TargetSchedule.td +++ b/llvm/include/llvm/Target/TargetSchedule.td @@ -584,3 +584,116 @@ class MemoryQueue { class LoadQueue : MemoryQueue; class StoreQueue : MemoryQueue; + +// The target instruction that FusionPredicate will be evaluated on. +class FusionTarget; +def first_fusion_target : FusionTarget; +def second_fusion_target : FusionTarget; +def both_fusion_target : FusionTarget; + +// Base class of FusionPredicate, etc. The avaliable variables are: +// * const TargetInstrInfo &TII +// * const TargetSubtargetInfo &STI +// * const MachineRegisterInfo &MRI +// * const MachineInstr *FirstMI +// * const MachineInstr &SecondMI +class FusionPredicate { + FusionTarget Target = target; +} +class FirstFusionPredicate: FusionPredicate; +class SecondFusionPredicate: FusionPredicate; +class BothFusionPredicate: FusionPredicate; + +// FusionPredicate with raw code predicate. +class FusionPredicateWithCode : FusionPredicate { + code Predicate = pred; +} + +// FusionPredicate with MCInstPredicate. +class FusionPredicateWithMCInstPredicate + : FusionPredicate { + MCInstPredicate Predicate = pred; +} +class FirstFusionPredicateWithMCInstPredicate + : FusionPredicateWithMCInstPredicate; +class SecondFusionPredicateWithMCInstPredicate + : FusionPredicateWithMCInstPredicate; +// The pred will be applied on both firstMI and secondMI. +class BothFusionPredicateWithMCInstPredicate + : FusionPredicateWithMCInstPredicate; + +// Tie firstOpIdx and secondOpIdx. The operand of `FirstMI` at position +// `firstOpIdx` should be the same as the operand of `SecondMI` at position +// `secondOpIdx`. +class TieReg : BothFusionPredicate { + int FirstOpIdx = firstOpIdx; + int SecondOpIdx = secondOpIdx; +} + +// A predicate for wildcard. The generated code will be like: +// ``` +// if (!FirstMI) +// return ReturnValue; +// ``` +class WildcardPred : FirstFusionPredicate { + bit ReturnValue = ret; +} +def WildcardFalse : WildcardPred<0>; +def WildcardTrue : WildcardPred<1>; + +// Indicates that the destination register of `FirstMI` should have one use if +// it is a virtual register. +class OneUsePred : FirstFusionPredicate; +def OneUse : OneUsePred; + +// Handled by MacroFusionPredicatorEmitter backend. +// The generated predicator will be like: +// ``` +// bool isNAME(const TargetInstrInfo &TII, +// const TargetSubtargetInfo &STI, +// const MachineInstr *FirstMI, +// const MachineInstr &SecondMI) { +// auto &MRI = SecondMI.getMF()->getRegInfo(); +// /* Predicates */ +// return true; +// } +// ``` +class Fusion predicates> { + list Predicates = predicates; +} + +// The generated predicator will be like: +// ``` +// bool isNAME(const TargetInstrInfo &TII, +// const TargetSubtargetInfo &STI, +// const MachineInstr *FirstMI, +// const MachineInstr &SecondMI) { +// auto &MRI = SecondMI.getMF()->getRegInfo(); +// /* Prolog */ +// /* Predicate for `SecondMI` */ +// /* Wildcard */ +// /* Predicate for `FirstMI` */ +// /* Check One Use */ +// /* Tie registers */ +// /* Epilog */ +// return true; +// } +// ``` +class SimpleFusion prolog = [], + list epilog = []> + : Fusion, + WildcardTrue, + FirstFusionPredicateWithMCInstPredicate, + SecondFusionPredicateWithMCInstPredicate< + CheckAny<[ + CheckIsVRegOperand<0>, + CheckSameRegOperand<0, 1> + ]>>, + OneUse, + TieReg<0, 1>, + ], + epilog)>; diff --git a/llvm/test/TableGen/MacroFusion.td b/llvm/test/TableGen/MacroFusion.td new file mode 100644 index 000000000000..f984a142839c --- /dev/null +++ b/llvm/test/TableGen/MacroFusion.td @@ -0,0 +1,97 @@ +// RUN: llvm-tblgen -gen-macro-fusion-pred -I %p/../../include %s | FileCheck %s --check-prefix=CHECK-PREDICATOR + +include "llvm/Target/Target.td" + +def TestInstrInfo : InstrInfo { } +def TestAsmWriter : AsmWriter { + int PassSubtarget = 1; +} + +def Test : Target { + let InstructionSet = TestInstrInfo; + let AssemblyWriters = [TestAsmWriter]; +} + +let Namespace = "Test" in { + foreach i = 0-32 in { + def X#i : Register<"x"#i>; + } + + def GPR : RegisterClass<"GPR", [i32], 32, (sequence "X%u", 0, 32)>; + + class TestInst : Instruction { + field bits<32> Inst; + field bits<32> SoftFail = 0; + let Size = 4; + let Inst = Opc; + let OutOperandList = (outs); + let InOperandList = (ins); + let AsmString = NAME; + } +} + +def Inst0 : TestInst<0>; +def Inst1 : TestInst<1>; + +def TestFusion: SimpleFusion, + CheckAll<[ + CheckOpcode<[Inst1]>, + CheckRegOperand<0, X0> + ]>>; + +// CHECK-PREDICATOR: #ifdef GET_Test_MACRO_FUSION_PRED_DECL +// CHECK-PREDICATOR-NEXT: #undef GET_Test_MACRO_FUSION_PRED_DECL +// CHECK-PREDICATOR-EMPTY: +// CHECK-PREDICATOR-NEXT: namespace llvm { +// CHECK-PREDICATOR-NEXT: bool isTestFusion(const TargetInstrInfo &, const TargetSubtargetInfo &, const MachineInstr *, const MachineInstr &); +// CHECK-PREDICATOR-NEXT: } // end namespace llvm +// CHECK-PREDICATOR-EMPTY: +// CHECK-PREDICATOR-NEXT: #endif + +// CHECK-PREDICATOR: #ifdef GET_Test_MACRO_FUSION_PRED_IMPL +// CHECK-PREDICATOR-NEXT: #undef GET_Test_MACRO_FUSION_PRED_IMPL +// CHECK-PREDICATOR-EMPTY: +// CHECK-PREDICATOR-NEXT: namespace llvm { +// CHECK-PREDICATOR-NEXT: bool isTestFusion( +// CHECK-PREDICATOR-NEXT: const TargetInstrInfo &TII, +// CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI, +// CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI, +// CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) { +// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo(); +// CHECK-PREDICATOR-NEXT: { +// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI; +// CHECK-PREDICATOR-NEXT: if (!( +// CHECK-PREDICATOR-NEXT: ( MI->getOpcode() == Test::Inst1 ) +// CHECK-PREDICATOR-NEXT: && MI->getOperand(0).getReg() == Test::X0 +// CHECK-PREDICATOR-NEXT: )) +// CHECK-PREDICATOR-NEXT: return false; +// CHECK-PREDICATOR-NEXT: } +// CHECK-PREDICATOR-NEXT: if (!FirstMI) +// CHECK-PREDICATOR-NEXT: return true; +// CHECK-PREDICATOR-NEXT: { +// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = FirstMI; +// CHECK-PREDICATOR-NEXT: if (( MI->getOpcode() != Test::Inst0 )) +// CHECK-PREDICATOR-NEXT: return false; +// CHECK-PREDICATOR-NEXT: } +// CHECK-PREDICATOR-NEXT: { +// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI; +// CHECK-PREDICATOR-NEXT: if (!( +// CHECK-PREDICATOR-NEXT: MI->getOperand(0).getReg().isVirtual() +// CHECK-PREDICATOR-NEXT: || MI->getOperand(0).getReg() == MI->getOperand(1).getReg() +// CHECK-PREDICATOR-NEXT: )) +// CHECK-PREDICATOR-NEXT: return false; +// CHECK-PREDICATOR-NEXT: } +// CHECK-PREDICATOR-NEXT: { +// CHECK-PREDICATOR-NEXT: Register FirstDest = FirstMI->getOperand(0).getReg(); +// CHECK-PREDICATOR-NEXT: if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest)) +// CHECK-PREDICATOR-NEXT: return false; +// CHECK-PREDICATOR-NEXT: } +// CHECK-PREDICATOR-NEXT: if (!(FirstMI->getOperand(0).isReg() && +// CHECK-PREDICATOR-NEXT: SecondMI.getOperand(1).isReg() && +// CHECK-PREDICATOR-NEXT: FirstMI->getOperand(0).getReg() == SecondMI.getOperand(1).getReg())) +// CHECK-PREDICATOR-NEXT: return false; +// CHECK-PREDICATOR-NEXT: return true; +// CHECK-PREDICATOR-NEXT: } +// CHECK-PREDICATOR-NEXT: } // end namespace llvm +// CHECK-PREDICATOR-EMPTY: +// CHECK-PREDICATOR-NEXT: #endif diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index 071ea3bc0705..f765cc36d3be 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -72,6 +72,7 @@ add_tablegen(llvm-tblgen LLVM PredicateExpander.cpp PseudoLoweringEmitter.cpp CompressInstEmitter.cpp + MacroFusionPredicatorEmitter.cpp RegisterBankEmitter.cpp RegisterInfoEmitter.cpp SearchableTableEmitter.cpp diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp new file mode 100644 index 000000000000..78dcd4471ae7 --- /dev/null +++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp @@ -0,0 +1,236 @@ +//===------ MacroFusionPredicatorEmitter.cpp - Generator for Fusion ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// MacroFusionPredicatorEmitter implements a TableGen-driven predicators +// generator for macro-op fusions. +// +// This TableGen backend processes `Fusion` definitions and generates +// predicators for checking if input instructions can be fused. These +// predicators can used in `MacroFusion` DAG mutation. +// +// The generated header file contains two parts: one for predicator +// declarations and one for predicator implementations. The user can get them +// by defining macro `GET__MACRO_FUSION_PRED_DECL` or +// `GET__MACRO_FUSION_PRED_IMPL` and then including the generated +// header file. +// +// The generated predicator will be like: +// +// ``` +// bool isNAME(const TargetInstrInfo &TII, +// const TargetSubtargetInfo &STI, +// const MachineInstr *FirstMI, +// const MachineInstr &SecondMI) { +// auto &MRI = SecondMI.getMF()->getRegInfo(); +// /* Predicates */ +// return true; +// } +// ``` +// +// The `Predicates` part is generated from a list of `FusionPredicate`, which +// can be predefined predicates, a raw code string or `MCInstPredicate` defined +// in TargetInstrPredicate.td. +// +//===---------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "PredicateExpander.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include +#include + +using namespace llvm; + +#define DEBUG_TYPE "macro-fusion-predicator" + +namespace { +class MacroFusionPredicatorEmitter { + RecordKeeper &Records; + CodeGenTarget Target; + + void emitMacroFusionDecl(std::vector Fusions, PredicateExpander &PE, + raw_ostream &OS); + void emitMacroFusionImpl(std::vector Fusions, PredicateExpander &PE, + raw_ostream &OS); + void emitPredicates(std::vector &FirstPredicate, + PredicateExpander &PE, raw_ostream &OS); + void emitFirstPredicate(Record *SecondPredicate, PredicateExpander &PE, + raw_ostream &OS); + void emitSecondPredicate(Record *SecondPredicate, PredicateExpander &PE, + raw_ostream &OS); + void emitBothPredicate(Record *Predicates, PredicateExpander &PE, + raw_ostream &OS); + +public: + MacroFusionPredicatorEmitter(RecordKeeper &R) : Records(R), Target(R) {} + + void run(raw_ostream &OS); +}; +} // End anonymous namespace. + +void MacroFusionPredicatorEmitter::emitMacroFusionDecl( + std::vector Fusions, PredicateExpander &PE, raw_ostream &OS) { + OS << "#ifdef GET_" << Target.getName() << "_MACRO_FUSION_PRED_DECL\n"; + OS << "#undef GET_" << Target.getName() << "_MACRO_FUSION_PRED_DECL\n\n"; + OS << "namespace llvm {\n"; + + for (Record *Fusion : Fusions) { + OS << "bool is" << Fusion->getName() << "(const TargetInstrInfo &, " + << "const TargetSubtargetInfo &, " + << "const MachineInstr *, " + << "const MachineInstr &);\n"; + } + + OS << "} // end namespace llvm\n"; + OS << "\n#endif\n"; +} + +void MacroFusionPredicatorEmitter::emitMacroFusionImpl( + std::vector Fusions, PredicateExpander &PE, raw_ostream &OS) { + OS << "#ifdef GET_" << Target.getName() << "_MACRO_FUSION_PRED_IMPL\n"; + OS << "#undef GET_" << Target.getName() << "_MACRO_FUSION_PRED_IMPL\n\n"; + OS << "namespace llvm {\n"; + + for (Record *Fusion : Fusions) { + std::vector Predicates = + Fusion->getValueAsListOfDefs("Predicates"); + + OS << "bool is" << Fusion->getName() << "(\n"; + OS.indent(4) << "const TargetInstrInfo &TII,\n"; + OS.indent(4) << "const TargetSubtargetInfo &STI,\n"; + OS.indent(4) << "const MachineInstr *FirstMI,\n"; + OS.indent(4) << "const MachineInstr &SecondMI) {\n"; + OS.indent(2) << "auto &MRI = SecondMI.getMF()->getRegInfo();\n"; + + emitPredicates(Predicates, PE, OS); + + OS.indent(2) << "return true;\n"; + OS << "}\n"; + } + + OS << "} // end namespace llvm\n"; + OS << "\n#endif\n"; +} + +void MacroFusionPredicatorEmitter::emitPredicates( + std::vector &Predicates, PredicateExpander &PE, raw_ostream &OS) { + for (Record *Predicate : Predicates) { + Record *Target = Predicate->getValueAsDef("Target"); + if (Target->getName() == "first_fusion_target") + emitFirstPredicate(Predicate, PE, OS); + else if (Target->getName() == "second_fusion_target") + emitSecondPredicate(Predicate, PE, OS); + else if (Target->getName() == "both_fusion_target") + emitBothPredicate(Predicate, PE, OS); + else + PrintFatalError(Target->getLoc(), + "Unsupported 'FusionTarget': " + Target->getName()); + } +} + +void MacroFusionPredicatorEmitter::emitFirstPredicate(Record *Predicate, + PredicateExpander &PE, + raw_ostream &OS) { + if (Predicate->isSubClassOf("WildcardPred")) { + OS.indent(2) << "if (!FirstMI)\n"; + OS.indent(2) << " return " + << (Predicate->getValueAsBit("ReturnValue") ? "true" : "false") + << ";\n"; + } else if (Predicate->isSubClassOf("OneUsePred")) { + OS.indent(2) << "{\n"; + OS.indent(4) << "Register FirstDest = FirstMI->getOperand(0).getReg();\n"; + OS.indent(4) + << "if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest))\n"; + OS.indent(4) << " return false;\n"; + OS.indent(2) << "}\n"; + } else if (Predicate->isSubClassOf( + "FirstFusionPredicateWithMCInstPredicate")) { + OS.indent(2) << "{\n"; + OS.indent(4) << "const MachineInstr *MI = FirstMI;\n"; + OS.indent(4) << "if ("; + PE.setNegatePredicate(true); + PE.setIndentLevel(3); + PE.expandPredicate(OS, Predicate->getValueAsDef("Predicate")); + OS << ")\n"; + OS.indent(4) << " return false;\n"; + OS.indent(2) << "}\n"; + } else { + PrintFatalError(Predicate->getLoc(), + "Unsupported predicate for first instruction: " + + Predicate->getType()->getAsString()); + } +} + +void MacroFusionPredicatorEmitter::emitSecondPredicate(Record *Predicate, + PredicateExpander &PE, + raw_ostream &OS) { + if (Predicate->isSubClassOf("SecondFusionPredicateWithMCInstPredicate")) { + OS.indent(2) << "{\n"; + OS.indent(4) << "const MachineInstr *MI = &SecondMI;\n"; + OS.indent(4) << "if ("; + PE.setNegatePredicate(true); + PE.setIndentLevel(3); + PE.expandPredicate(OS, Predicate->getValueAsDef("Predicate")); + OS << ")\n"; + OS.indent(4) << " return false;\n"; + OS.indent(2) << "}\n"; + } else { + PrintFatalError(Predicate->getLoc(), + "Unsupported predicate for first instruction: " + + Predicate->getType()->getAsString()); + } +} + +void MacroFusionPredicatorEmitter::emitBothPredicate(Record *Predicate, + PredicateExpander &PE, + raw_ostream &OS) { + if (Predicate->isSubClassOf("FusionPredicateWithCode")) + OS << Predicate->getValueAsString("Predicate"); + else if (Predicate->isSubClassOf("BothFusionPredicateWithMCInstPredicate")) { + Record *MCPred = Predicate->getValueAsDef("Predicate"); + emitFirstPredicate(MCPred, PE, OS); + emitSecondPredicate(MCPred, PE, OS); + } else if (Predicate->isSubClassOf("TieReg")) { + int FirstOpIdx = Predicate->getValueAsInt("FirstOpIdx"); + int SecondOpIdx = Predicate->getValueAsInt("SecondOpIdx"); + OS.indent(2) << "if (!(FirstMI->getOperand(" << FirstOpIdx + << ").isReg() &&\n"; + OS.indent(2) << " SecondMI.getOperand(" << SecondOpIdx + << ").isReg() &&\n"; + OS.indent(2) << " FirstMI->getOperand(" << FirstOpIdx + << ").getReg() == SecondMI.getOperand(" << SecondOpIdx + << ").getReg()))\n"; + OS.indent(2) << " return false;\n"; + } else + PrintFatalError(Predicate->getLoc(), + "Unsupported predicate for both instruction: " + + Predicate->getType()->getAsString()); +} + +void MacroFusionPredicatorEmitter::run(raw_ostream &OS) { + // Emit file header. + emitSourceFileHeader("Macro Fusion Predicators", OS); + + PredicateExpander PE(Target.getName()); + PE.setByRef(false); + PE.setExpandForMC(false); + + std::vector Fusions = Records.getAllDerivedDefinitions("Fusion"); + // Sort macro fusions by name. + sort(Fusions, LessRecord()); + emitMacroFusionDecl(Fusions, PE, OS); + OS << "\n"; + emitMacroFusionImpl(Fusions, PE, OS); +} + +static TableGen::Emitter::OptClass + X("gen-macro-fusion-pred", "Generate macro fusion predicators."); diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/PredicateExpander.cpp index 8f96d3307ded..d3a73e02cd91 100644 --- a/llvm/utils/TableGen/PredicateExpander.cpp +++ b/llvm/utils/TableGen/PredicateExpander.cpp @@ -194,6 +194,11 @@ void PredicateExpander::expandCheckIsRegOperand(raw_ostream &OS, int OpIndex) { << "getOperand(" << OpIndex << ").isReg() "; } +void PredicateExpander::expandCheckIsVRegOperand(raw_ostream &OS, int OpIndex) { + OS << (shouldNegate() ? "!" : "") << "MI" << (isByRef() ? "." : "->") + << "getOperand(" << OpIndex << ").getReg().isVirtual()"; +} + void PredicateExpander::expandCheckIsImmOperand(raw_ostream &OS, int OpIndex) { OS << (shouldNegate() ? "!" : "") << "MI" << (isByRef() ? "." : "->") << "getOperand(" << OpIndex << ").isImm() "; @@ -319,6 +324,9 @@ void PredicateExpander::expandPredicate(raw_ostream &OS, const Record *Rec) { if (Rec->isSubClassOf("CheckIsRegOperand")) return expandCheckIsRegOperand(OS, Rec->getValueAsInt("OpIndex")); + if (Rec->isSubClassOf("CheckIsVRegOperand")) + return expandCheckIsVRegOperand(OS, Rec->getValueAsInt("OpIndex")); + if (Rec->isSubClassOf("CheckIsImmOperand")) return expandCheckIsImmOperand(OS, Rec->getValueAsInt("OpIndex")); diff --git a/llvm/utils/TableGen/PredicateExpander.h b/llvm/utils/TableGen/PredicateExpander.h index 27f049a715aa..cfb0a3d51e67 100644 --- a/llvm/utils/TableGen/PredicateExpander.h +++ b/llvm/utils/TableGen/PredicateExpander.h @@ -75,6 +75,7 @@ public: bool IsCheckAll); void expandTIIFunctionCall(raw_ostream &OS, StringRef MethodName); void expandCheckIsRegOperand(raw_ostream &OS, int OpIndex); + void expandCheckIsVRegOperand(raw_ostream &OS, int OpIndex); void expandCheckIsImmOperand(raw_ostream &OS, int OpIndex); void expandCheckInvalidRegOperand(raw_ostream &OS, int OpIndex); void expandCheckFunctionPredicate(raw_ostream &OS, StringRef MCInstFn, -- GitLab From 59569eb756265b2a5d9d96f6c6c5ee1a3c371c4f Mon Sep 17 00:00:00 2001 From: Arseniy Obolenskiy Date: Fri, 5 Jan 2024 22:49:21 +0800 Subject: [PATCH 120/728] [mlir] Fix support for loop normalization with integer indices (#76566) Choose correct type for updated loop boundaries after scf loop normalization, do not force chosen type to IndexType --- mlir/lib/Dialect/SCF/Utils/Utils.cpp | 7 ++++-- mlir/test/Dialect/SCF/transform-ops.mlir | 30 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp index e85825595e3c..a2043c647d49 100644 --- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp +++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp @@ -502,9 +502,12 @@ static LoopParams normalizeLoop(OpBuilder &boundsBuilder, Value newLowerBound = isZeroBased ? lowerBound - : boundsBuilder.create(loc, 0); + : boundsBuilder.create( + loc, boundsBuilder.getZeroAttr(lowerBound.getType())); Value newStep = - isStepOne ? step : boundsBuilder.create(loc, 1); + isStepOne ? step + : boundsBuilder.create( + loc, boundsBuilder.getIntegerAttr(step.getType(), 1)); // Insert code computing the value of the original loop induction variable // from the "normalized" one. diff --git a/mlir/test/Dialect/SCF/transform-ops.mlir b/mlir/test/Dialect/SCF/transform-ops.mlir index 93ebf67f8b71..a945758143c6 100644 --- a/mlir/test/Dialect/SCF/transform-ops.mlir +++ b/mlir/test/Dialect/SCF/transform-ops.mlir @@ -270,3 +270,33 @@ module attributes {transform.with_named_sequence} { transform.yield } } + +// ----- + +// CHECK-LABEL: func @coalesce_i32_loops( + +// This test checks for loop coalescing success for non-index loop boundaries and step type +func.func @coalesce_i32_loops() { + %0 = arith.constant 0 : i32 + %1 = arith.constant 128 : i32 + %2 = arith.constant 2 : i32 + %3 = arith.constant 64 : i32 + // CHECK-DAG: %[[C0_I32:.*]] = arith.constant 0 : i32 + // CHECK-DAG: %[[C1_I32:.*]] = arith.constant 1 : i32 + // CHECK: scf.for %[[ARG0:.*]] = %[[C0_I32]] to {{.*}} step %[[C1_I32]] : i32 + scf.for %i = %0 to %1 step %2 : i32 { + scf.for %j = %0 to %3 step %2 : i32 { + arith.addi %i, %j : i32 + } + } {coalesce} + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["scf.for"]} attributes {coalesce} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.cast %0 : !transform.any_op to !transform.op<"scf.for"> + %2 = transform.loop.coalesce %1: (!transform.op<"scf.for">) -> (!transform.op<"scf.for">) + transform.yield + } +} -- GitLab From 0f8adc8d30d2cd4a9fcf8455b64a1d66fa971339 Mon Sep 17 00:00:00 2001 From: erichkeane Date: Fri, 5 Jan 2024 06:49:07 -0800 Subject: [PATCH 121/728] [OpenACC] Fix comments on OpenACC enum to use /// --- clang/include/clang/Basic/OpenACCKinds.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h index 3117d584d347..54a29f8428e9 100644 --- a/clang/include/clang/Basic/OpenACCKinds.h +++ b/clang/include/clang/Basic/OpenACCKinds.h @@ -72,25 +72,25 @@ enum class OpenACCAtomicKind { /// Represents the kind of an OpenACC clause. enum class OpenACCClauseKind { - // 'finalize' clause, allowed on 'exit data' directive. + /// 'finalize' clause, allowed on 'exit data' directive. Finalize, - // 'if_present' clause, allowed on 'host_data' and 'update' directives. + /// 'if_present' clause, allowed on 'host_data' and 'update' directives. IfPresent, - // 'seq' clause, allowed on 'loop' and 'routine' directives. + /// 'seq' clause, allowed on 'loop' and 'routine' directives. Seq, - // 'independent' clause, allowed on 'loop' directives. + /// 'independent' clause, allowed on 'loop' directives. Independent, - // 'auto' clause, allowed on 'loop' directives. + /// 'auto' clause, allowed on 'loop' directives. Auto, - // 'worker' clause, allowed on 'loop' and 'routine' directives. + /// 'worker' clause, allowed on 'loop' and 'routine' directives. Worker, - // 'vector' clause, allowed on 'loop' and 'routine' directives. Takes no - // arguments for 'routine', so the 'loop' version is not yet implemented - // completely. + /// 'vector' clause, allowed on 'loop' and 'routine' directives. Takes no + /// arguments for 'routine', so the 'loop' version is not yet implemented + /// completely. Vector, - // 'nohost' clause, allowed on 'routine' directives. + /// 'nohost' clause, allowed on 'routine' directives. NoHost, - // Represents an invalid clause, for the purposes of parsing. + /// Represents an invalid clause, for the purposes of parsing. Invalid, }; } // namespace clang -- GitLab From 736cc0cbf0107f6a1678a5495e1931733ab57393 Mon Sep 17 00:00:00 2001 From: Nishant Mittal Date: Fri, 5 Jan 2024 20:19:51 +0530 Subject: [PATCH 122/728] [libc][math] fix failing nanl_test build (#77102) --- libc/test/src/math/smoke/nanl_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/test/src/math/smoke/nanl_test.cpp b/libc/test/src/math/smoke/nanl_test.cpp index c0e954016112..0a57b567916c 100644 --- a/libc/test/src/math/smoke/nanl_test.cpp +++ b/libc/test/src/math/smoke/nanl_test.cpp @@ -28,7 +28,7 @@ public: long double result = LIBC_NAMESPACE::nanl(input_str); auto actual_fp = LIBC_NAMESPACE::fputil::FPBits(result); auto expected_fp = LIBC_NAMESPACE::fputil::FPBits(bits); - EXPECT_EQ(actual_fp.bits, expected_fp.bits); + EXPECT_EQ(actual_fp.uintval(), expected_fp.uintval()); }; }; -- GitLab From 10b03e66629aedad79a804e22d23b575077303b3 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 5 Jan 2024 15:11:47 +0000 Subject: [PATCH 123/728] [RemoveDIs] Handle DPValues in FastISel (#76952) The change is fairly mechanical: 1. Factor code from `FastISel::selectIntrinsicCall`, which converts debug intrinsics into debug instructions, into functions (NFC). 2. Call those functions for DPValues attached to instructions too. The test updates look the same as other RemoveDIs changes: re-run the tests with `--try-experimental-debuginfo-iterators`, which checks the output is identical using the new debug info format (if it has been enabled in the cmake configuration). Depends on #76941 (otherwise some modified tests spuriously fail). --- llvm/include/llvm/CodeGen/FastISel.h | 14 + llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 323 ++++++++++-------- .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 4 + .../AArch64/fast-isel-branch-uncond-debug.ll | 3 +- llvm/test/CodeGen/AArch64/fast-isel-dbg.ll | 2 + .../AArch64/fastisel-debugvalue-undef.ll | 1 + llvm/test/CodeGen/ARM/debug-info-blocks.ll | 1 + llvm/test/CodeGen/Generic/csw-debug-assert.ll | 1 + .../CodeGen/X86/2010-08-04-StackVariable.ll | 1 + .../CodeGen/X86/DbgValueOtherTargets.test | 3 + .../CodeGen/X86/fast-isel-dbg-value-alloca.ll | 2 + llvm/test/CodeGen/X86/fold-sext-trunc.ll | 4 + llvm/test/CodeGen/X86/fold-zext-trunc.ll | 4 + llvm/test/CodeGen/X86/label-heapallocsite.ll | 3 + .../X86/machine-outliner-disubprogram.ll | 1 + .../CodeGen/X86/pr53243-tail-call-fastisel.ll | 1 + llvm/test/CodeGen/X86/sink-local-value.ll | 1 + .../DebugInfo/AArch64/cfi-eof-prologue.ll | 1 + llvm/test/DebugInfo/AArch64/frameindices.ll | 1 + llvm/test/DebugInfo/ARM/split-complex.ll | 7 +- .../DebugInfo/COFF/class-options-common.ll | 5 + llvm/test/DebugInfo/COFF/cpp-mangling.ll | 5 +- llvm/test/DebugInfo/COFF/enum-co.ll | 3 + llvm/test/DebugInfo/COFF/function-options.ll | 3 + llvm/test/DebugInfo/COFF/global_visibility.ll | 1 + llvm/test/DebugInfo/COFF/globals.ll | 8 + llvm/test/DebugInfo/COFF/lambda.ll | 3 + llvm/test/DebugInfo/COFF/lines-bb-start.ll | 1 + llvm/test/DebugInfo/COFF/nrvo.ll | 3 + llvm/test/DebugInfo/COFF/numeric-leaves.ll | 5 + .../test/DebugInfo/COFF/parent-type-scopes.ll | 3 + .../test/DebugInfo/COFF/purge-typedef-udts.ll | 1 + llvm/test/DebugInfo/COFF/thunk.ll | 7 +- llvm/test/DebugInfo/COFF/type-quals.ll | 3 + llvm/test/DebugInfo/COFF/types-cvarargs.ll | 3 + llvm/test/DebugInfo/COFF/types-integer-old.ll | 1 + .../COFF/types-method-ref-qualifiers.ll | 3 + .../DebugInfo/Generic/2010-10-01-crash.ll | 1 + llvm/test/DebugInfo/Generic/PR20038.ll | 1 + .../DebugInfo/Generic/dead-argument-order.ll | 1 + .../DebugInfo/Generic/discriminated-union.ll | 3 + llvm/test/DebugInfo/Generic/disubrange_vla.ll | 7 + llvm/test/DebugInfo/Generic/enum-types.ll | 1 + llvm/test/DebugInfo/Generic/enum.ll | 3 + .../Generic/import-inlined-declaration.ll | 2 + llvm/test/DebugInfo/Generic/inlined-vars.ll | 2 + .../DebugInfo/Generic/recursive_inlining.ll | 1 + .../Generic/univariant-discriminated-union.ll | 3 + llvm/test/DebugInfo/Mips/delay-slot.ll | 1 + llvm/test/DebugInfo/X86/aligned_stack_var.ll | 3 + llvm/test/DebugInfo/X86/arguments.ll | 3 + llvm/test/DebugInfo/X86/asan_debug_info.ll | 2 +- llvm/test/DebugInfo/X86/byvalstruct.ll | 2 + llvm/test/DebugInfo/X86/convert-linked.ll | 1 + llvm/test/DebugInfo/X86/dbg-declare-arg.ll | 1 + .../DebugInfo/X86/dbg-declare-inalloca.ll | 5 + llvm/test/DebugInfo/X86/dbg-declare.ll | 5 + llvm/test/DebugInfo/X86/dbg_value_direct.ll | 1 + .../X86/debug-info-template-parameter.ll | 1 + .../X86/debug_value_list_selectiondag.ll | 6 + llvm/test/DebugInfo/X86/double-declare.ll | 1 + llvm/test/DebugInfo/X86/fi-piece.ll | 1 + .../DebugInfo/X86/implicit_value-double.ll | 4 + .../DebugInfo/X86/instr-ref-opt-levels.ll | 11 + .../DebugInfo/X86/instr-ref-selectiondag.ll | 15 + .../X86/missing-abstract-variable.ll | 2 +- llvm/test/DebugInfo/X86/parameters.ll | 1 + llvm/test/DebugInfo/X86/pieces-1.ll | 1 + llvm/test/DebugInfo/X86/reference-argument.ll | 2 + .../test/DebugInfo/X86/spill-indirect-nrvo.ll | 6 +- llvm/test/DebugInfo/X86/sret.ll | 8 +- llvm/test/DebugInfo/X86/subreg.ll | 1 + llvm/test/DebugInfo/X86/subregisters.ll | 3 + llvm/test/DebugInfo/X86/vla.ll | 1 + 74 files changed, 398 insertions(+), 151 deletions(-) diff --git a/llvm/include/llvm/CodeGen/FastISel.h b/llvm/include/llvm/CodeGen/FastISel.h index dc2931b40d35..0f17e51f0b7a 100644 --- a/llvm/include/llvm/CodeGen/FastISel.h +++ b/llvm/include/llvm/CodeGen/FastISel.h @@ -319,6 +319,10 @@ public: /// Reset InsertPt to the given old insert position. void leaveLocalValueArea(SavePoint Old); + /// Target-independent lowering of non-instruction debug info associated with + /// this instruction. + void handleDbgInfo(const Instruction *II); + protected: explicit FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, @@ -518,6 +522,16 @@ protected: return MF->getFunction().hasOptSize(); } + /// Target-independent lowering of debug information. Returns false if the + /// debug information couldn't be lowered and was instead discarded. + virtual bool lowerDbgValue(const Value *V, DIExpression *Expr, + DILocalVariable *Var, const DebugLoc &DL); + + /// Target-independent lowering of debug information. Returns false if the + /// debug information couldn't be lowered and was instead discarded. + virtual bool lowerDbgDeclare(const Value *V, DIExpression *Expr, + DILocalVariable *Var, const DebugLoc &DL); + private: /// Handle PHI nodes in successor blocks. /// diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 09e260394262..6d80b282a1ed 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1180,6 +1180,184 @@ bool FastISel::selectCall(const User *I) { return lowerCall(Call); } +void FastISel::handleDbgInfo(const Instruction *II) { + if (!II->hasDbgValues()) + return; + + // Clear any metadata. + MIMD = MIMetadata(); + + // Reverse order of debug records, because fast-isel walks through backwards. + for (DPValue &DPV : llvm::reverse(II->getDbgValueRange())) { + flushLocalValueMap(); + recomputeInsertPt(); + + Value *V = nullptr; + if (!DPV.hasArgList()) + V = DPV.getVariableLocationOp(0); + + bool Res = false; + if (DPV.getType() == DPValue::LocationType::Value) { + Res = lowerDbgValue(V, DPV.getExpression(), DPV.getVariable(), + DPV.getDebugLoc()); + } else { + assert(DPV.getType() == DPValue::LocationType::Declare); + if (FuncInfo.PreprocessedDPVDeclares.contains(&DPV)) + continue; + Res = lowerDbgDeclare(V, DPV.getExpression(), DPV.getVariable(), + DPV.getDebugLoc()); + } + + if (!Res) + LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DPV << "\n";); + } +} + +bool FastISel::lowerDbgValue(const Value *V, DIExpression *Expr, + DILocalVariable *Var, const DebugLoc &DL) { + // This form of DBG_VALUE is target-independent. + const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); + if (!V || isa(V)) { + // DI is either undef or cannot produce a valid DBG_VALUE, so produce an + // undef DBG_VALUE to terminate any prior location. + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, false, 0U, Var, Expr); + return true; + } + if (const auto *CI = dyn_cast(V)) { + // See if there's an expression to constant-fold. + if (Expr) + std::tie(Expr, CI) = Expr->constantFold(CI); + if (CI->getBitWidth() > 64) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addCImm(CI) + .addImm(0U) + .addMetadata(Var) + .addMetadata(Expr); + else + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addImm(CI->getZExtValue()) + .addImm(0U) + .addMetadata(Var) + .addMetadata(Expr); + return true; + } + if (const auto *CF = dyn_cast(V)) { + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addFPImm(CF) + .addImm(0U) + .addMetadata(Var) + .addMetadata(Expr); + return true; + } + if (const auto *Arg = dyn_cast(V); + Arg && Expr && Expr->isEntryValue()) { + // As per the Verifier, this case is only valid for swift async Args. + assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync)); + + Register Reg = getRegForValue(Arg); + for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins()) + if (Reg == VirtReg || Reg == PhysReg) { + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, false /*IsIndirect*/, + PhysReg, Var, Expr); + return true; + } + + LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but " + "couldn't find a physical register\n"); + return false; + } + if (auto SI = FuncInfo.StaticAllocaMap.find(dyn_cast(V)); + SI != FuncInfo.StaticAllocaMap.end()) { + MachineOperand FrameIndexOp = MachineOperand::CreateFI(SI->second); + bool IsIndirect = false; + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, IsIndirect, FrameIndexOp, + Var, Expr); + return true; + } + if (Register Reg = lookUpRegForValue(V)) { + // FIXME: This does not handle register-indirect values at offset 0. + if (!FuncInfo.MF->useDebugInstrRef()) { + bool IsIndirect = false; + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, IsIndirect, Reg, Var, + Expr); + return true; + } + // If using instruction referencing, produce this as a DBG_INSTR_REF, + // to be later patched up by finalizeDebugInstrRefs. + SmallVector MOs({MachineOperand::CreateReg( + /* Reg */ Reg, /* isDef */ false, /* isImp */ false, + /* isKill */ false, /* isDead */ false, + /* isUndef */ false, /* isEarlyClobber */ false, + /* SubReg */ 0, /* isDebug */ true)}); + SmallVector Ops({dwarf::DW_OP_LLVM_arg, 0}); + auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs, + Var, NewExpr); + return true; + } + return false; +} + +bool FastISel::lowerDbgDeclare(const Value *Address, DIExpression *Expr, + DILocalVariable *Var, const DebugLoc &DL) { + if (!Address || isa(Address)) { + LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n"); + return false; + } + + std::optional Op; + if (Register Reg = lookUpRegForValue(Address)) + Op = MachineOperand::CreateReg(Reg, false); + + // If we have a VLA that has a "use" in a metadata node that's then used + // here but it has no other uses, then we have a problem. E.g., + // + // int foo (const int *x) { + // char a[*x]; + // return 0; + // } + // + // If we assign 'a' a vreg and fast isel later on has to use the selection + // DAG isel, it will want to copy the value to the vreg. However, there are + // no uses, which goes counter to what selection DAG isel expects. + if (!Op && !Address->use_empty() && isa(Address) && + (!isa(Address) || + !FuncInfo.StaticAllocaMap.count(cast(Address)))) + Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address), + false); + + if (Op) { + assert(Var->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); + if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) { + // If using instruction referencing, produce this as a DBG_INSTR_REF, + // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto + // the expression, we don't have an "indirect" flag in DBG_INSTR_REF. + SmallVector Ops( + {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref}); + auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, *Op, + Var, NewExpr); + return true; + } + + // A dbg.declare describes the address of a source variable, so lower it + // into an indirect DBG_VALUE. + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, *Op, Var, + Expr); + return true; + } + + // We can't yet handle anything else here because it would require + // generating code, thus altering codegen because of debug info. + LLVM_DEBUG( + dbgs() << "Dropping debug info (no materialized reg for address)\n"); + return false; +} + bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { switch (II->getIntrinsicID()) { default: @@ -1209,153 +1387,28 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { return true; const Value *Address = DI->getAddress(); - if (!Address || isa(Address)) { - LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI - << " (bad/undef address)\n"); - return true; - } + if (!lowerDbgDeclare(Address, DI->getExpression(), DI->getVariable(), + MIMD.getDL())) + LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI); - std::optional Op; - if (Register Reg = lookUpRegForValue(Address)) - Op = MachineOperand::CreateReg(Reg, false); - - // If we have a VLA that has a "use" in a metadata node that's then used - // here but it has no other uses, then we have a problem. E.g., - // - // int foo (const int *x) { - // char a[*x]; - // return 0; - // } - // - // If we assign 'a' a vreg and fast isel later on has to use the selection - // DAG isel, it will want to copy the value to the vreg. However, there are - // no uses, which goes counter to what selection DAG isel expects. - if (!Op && !Address->use_empty() && isa(Address) && - (!isa(Address) || - !FuncInfo.StaticAllocaMap.count(cast(Address)))) - Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address), - false); - - if (Op) { - assert(DI->getVariable()->isValidLocationForIntrinsic(MIMD.getDL()) && - "Expected inlined-at fields to agree"); - if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) { - // If using instruction referencing, produce this as a DBG_INSTR_REF, - // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto - // the expression, we don't have an "indirect" flag in DBG_INSTR_REF. - SmallVector Ops( - {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref}); - auto *NewExpr = DIExpression::prependOpcodes(DI->getExpression(), Ops); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), - TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, *Op, - DI->getVariable(), NewExpr); - } else { - // A dbg.declare describes the address of a source variable, so lower it - // into an indirect DBG_VALUE. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), - TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, *Op, - DI->getVariable(), DI->getExpression()); - } - } else { - // We can't yet handle anything else here because it would require - // generating code, thus altering codegen because of debug info. - LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI - << " (no materialized reg for address)\n"); - } return true; } case Intrinsic::dbg_value: { // This form of DBG_VALUE is target-independent. const DbgValueInst *DI = cast(II); - const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); const Value *V = DI->getValue(); DIExpression *Expr = DI->getExpression(); DILocalVariable *Var = DI->getVariable(); + if (DI->hasArgList()) + // Signal that we don't have a location for this. + V = nullptr; + assert(Var->isValidLocationForIntrinsic(MIMD.getDL()) && "Expected inlined-at fields to agree"); - if (!V || isa(V) || DI->hasArgList()) { - // DI is either undef or cannot produce a valid DBG_VALUE, so produce an - // undef DBG_VALUE to terminate any prior location. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, false, 0U, - Var, Expr); - return true; - } - if (const auto *CI = dyn_cast(V)) { - // See if there's an expression to constant-fold. - if (Expr) - std::tie(Expr, CI) = Expr->constantFold(CI); - if (CI->getBitWidth() > 64) - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II) - .addCImm(CI) - .addImm(0U) - .addMetadata(Var) - .addMetadata(Expr); - else - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II) - .addImm(CI->getZExtValue()) - .addImm(0U) - .addMetadata(Var) - .addMetadata(Expr); - return true; - } - if (const auto *CF = dyn_cast(V)) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II) - .addFPImm(CF) - .addImm(0U) - .addMetadata(Var) - .addMetadata(Expr); - return true; - } - if (const auto *Arg = dyn_cast(V); - Arg && Expr && Expr->isEntryValue()) { - // As per the Verifier, this case is only valid for swift async Args. - assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync)); - - Register Reg = getRegForValue(Arg); - for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins()) - if (Reg == VirtReg || Reg == PhysReg) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, - false /*IsIndirect*/, PhysReg, Var, Expr); - return true; - } - LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but " - "couldn't find a physical register\n" - << *DI << "\n"); - return true; - } - if (auto SI = FuncInfo.StaticAllocaMap.find(dyn_cast(V)); - SI != FuncInfo.StaticAllocaMap.end()) { - MachineOperand FrameIndexOp = MachineOperand::CreateFI(SI->second); - bool IsIndirect = false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, IsIndirect, - FrameIndexOp, Var, Expr); - return true; - } - if (Register Reg = lookUpRegForValue(V)) { - // FIXME: This does not handle register-indirect values at offset 0. - if (!FuncInfo.MF->useDebugInstrRef()) { - bool IsIndirect = false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, IsIndirect, - Reg, Var, Expr); - return true; - } - // If using instruction referencing, produce this as a DBG_INSTR_REF, - // to be later patched up by finalizeDebugInstrRefs. - SmallVector MOs({MachineOperand::CreateReg( - /* Reg */ Reg, /* isDef */ false, /* isImp */ false, - /* isKill */ false, /* isDead */ false, - /* isUndef */ false, /* isEarlyClobber */ false, - /* SubReg */ 0, /* isDebug */ true)}); - SmallVector Ops({dwarf::DW_OP_LLVM_arg, 0}); - auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), - TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs, - Var, NewExpr); - return true; - } - // We don't know how to handle other cases, so we drop. - LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); + if (!lowerDbgValue(V, Expr, Var, MIMD.getDL())) + LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); + return true; } case Intrinsic::dbg_label: { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index f28211ac113c..99bb3d875d4f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -1614,6 +1614,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { if (isFoldedOrDeadInstruction(Inst, *FuncInfo) || ElidedArgCopyInstrs.count(Inst)) { --NumFastIselRemaining; + FastIS->handleDbgInfo(Inst); continue; } @@ -1625,6 +1626,8 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { if (FastIS->selectInstruction(Inst)) { --NumFastIselRemaining; ++NumFastIselSuccess; + + FastIS->handleDbgInfo(Inst); // If fast isel succeeded, skip over all the folded instructions, and // then see if there is a load right before the selected instructions. // Try to fold the load if so. @@ -1640,6 +1643,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // If we succeeded, don't re-select the load. LLVM_DEBUG(dbgs() << "FastISel folded load: " << *BeforeInst << "\n"); + FastIS->handleDbgInfo(BeforeInst); BI = std::next(BasicBlock::const_iterator(BeforeInst)); --NumFastIselRemaining; ++NumFastIselSuccess; diff --git a/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll b/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll index 1d213945966a..e1614d322b93 100644 --- a/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll +++ b/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple=aarch64 -O1 -opt-bisect-limit=2 -o - %s 2> /dev/null | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=aarch64 -O1 -opt-bisect-limit=2 -o - %s 2> /dev/null | FileCheck %s define dso_local i32 @a() #0 !dbg !7 { entry: @@ -41,4 +42,4 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) #2 !18 = distinct !{!18, !19, !20} !19 = !DILocation(line: 3, column: 3, scope: !17) !20 = !DILocation(line: 4, column: 5, scope: !17) - \ No newline at end of file + diff --git a/llvm/test/CodeGen/AArch64/fast-isel-dbg.ll b/llvm/test/CodeGen/AArch64/fast-isel-dbg.ll index 602d454044d4..0c1a693089cb 100644 --- a/llvm/test/CodeGen/AArch64/fast-isel-dbg.ll +++ b/llvm/test/CodeGen/AArch64/fast-isel-dbg.ll @@ -1,4 +1,6 @@ ; RUN: llc -o - %s -fast-isel -stop-before=finalize-isel | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -o - %s -fast-isel -stop-before=finalize-isel | FileCheck %s + ; Make sure fast-isel produces DBG_VALUE instructions even if no debug printer ; is scheduled because of -stop-before. target triple="aarch64--" diff --git a/llvm/test/CodeGen/AArch64/fastisel-debugvalue-undef.ll b/llvm/test/CodeGen/AArch64/fastisel-debugvalue-undef.ll index cccede6a5045..c30a01baedaf 100644 --- a/llvm/test/CodeGen/AArch64/fastisel-debugvalue-undef.ll +++ b/llvm/test/CodeGen/AArch64/fastisel-debugvalue-undef.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -fast-isel=1 -o - -print-after="finalize-isel" %s 2>&1 | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 -fast-isel=1 -o - -print-after="finalize-isel" %s 2>&1 | FileCheck %s ; Check that we emit a DBG_VALUE for the `@llvm.dbg.value` which has `undef` has first arg. diff --git a/llvm/test/CodeGen/ARM/debug-info-blocks.ll b/llvm/test/CodeGen/ARM/debug-info-blocks.ll index 1c9ffb1775aa..8ef341faed6b 100644 --- a/llvm/test/CodeGen/ARM/debug-info-blocks.ll +++ b/llvm/test/CodeGen/ARM/debug-info-blocks.ll @@ -1,4 +1,5 @@ ; RUN: llc -filetype=obj -O0 < %s | llvm-dwarfdump -v - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -filetype=obj -O0 < %s | llvm-dwarfdump -v - | FileCheck %s ; debug_info content ; CHECK: DW_AT_name {{.*}} "foobar_func_block_invoke_0" diff --git a/llvm/test/CodeGen/Generic/csw-debug-assert.ll b/llvm/test/CodeGen/Generic/csw-debug-assert.ll index 084cbfa79679..85a8d8bb1d98 100644 --- a/llvm/test/CodeGen/Generic/csw-debug-assert.ll +++ b/llvm/test/CodeGen/Generic/csw-debug-assert.ll @@ -2,6 +2,7 @@ ; REQUIRES: asserts ; REQUIRES: x86_64-linux ; RUN: llc -O1 -regalloc=pbqp < %s | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O1 -regalloc=pbqp < %s | FileCheck %s target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/X86/2010-08-04-StackVariable.ll b/llvm/test/CodeGen/X86/2010-08-04-StackVariable.ll index 198c0d4806c5..e95b3b4884b4 100644 --- a/llvm/test/CodeGen/X86/2010-08-04-StackVariable.ll +++ b/llvm/test/CodeGen/X86/2010-08-04-StackVariable.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=x86_64-apple-darwin < %s | grep DW_OP_breg7 +; RUN: llc --try-experimental-debuginfo-iterators -O0 -mtriple=x86_64-apple-darwin < %s | grep DW_OP_breg7 ; Use DW_OP_breg7 in variable's location expression if the variable is in a stack slot. %struct.SVal = type { ptr, i32 } diff --git a/llvm/test/CodeGen/X86/DbgValueOtherTargets.test b/llvm/test/CodeGen/X86/DbgValueOtherTargets.test index 02f56823fd1d..29497d33f075 100644 --- a/llvm/test/CodeGen/X86/DbgValueOtherTargets.test +++ b/llvm/test/CodeGen/X86/DbgValueOtherTargets.test @@ -1,2 +1,5 @@ RUN: llc -O0 -mtriple=i686-- -asm-verbose < %S/../Inputs/DbgValueOtherTargets.ll | FileCheck %S/../Inputs/DbgValueOtherTargets.ll RUN: llc -O0 -mtriple=x86_64-- -asm-verbose < %S/../Inputs/DbgValueOtherTargets.ll | FileCheck %S/../Inputs/DbgValueOtherTargets.ll + +RUN: llc --try-experimental-debuginfo-iterators -O0 -mtriple=i686-- -asm-verbose < %S/../Inputs/DbgValueOtherTargets.ll | FileCheck %S/../Inputs/DbgValueOtherTargets.ll +RUN: llc --try-experimental-debuginfo-iterators -O0 -mtriple=x86_64-- -asm-verbose < %S/../Inputs/DbgValueOtherTargets.ll | FileCheck %S/../Inputs/DbgValueOtherTargets.ll \ No newline at end of file diff --git a/llvm/test/CodeGen/X86/fast-isel-dbg-value-alloca.ll b/llvm/test/CodeGen/X86/fast-isel-dbg-value-alloca.ll index 77f883ee9bd3..bdb5239d289e 100644 --- a/llvm/test/CodeGen/X86/fast-isel-dbg-value-alloca.ll +++ b/llvm/test/CodeGen/X86/fast-isel-dbg-value-alloca.ll @@ -1,6 +1,8 @@ ; RUN: llc -fast-isel -fast-isel-abort=1 -mtriple=x86_64-unknown-unknown -stop-after=finalize-isel %s -o - | \ ; RUN: FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -fast-isel -fast-isel-abort=1 -mtriple=x86_64-unknown-unknown -stop-after=finalize-isel %s -o - | FileCheck %s + define void @foo(ptr noalias nocapture %arg) !dbg !38 { %k.debug = alloca ptr, align 8 store ptr %arg, ptr %k.debug, align 8, !dbg !70 diff --git a/llvm/test/CodeGen/X86/fold-sext-trunc.ll b/llvm/test/CodeGen/X86/fold-sext-trunc.ll index c29ca2204a6a..9963bb7b030d 100644 --- a/llvm/test/CodeGen/X86/fold-sext-trunc.ll +++ b/llvm/test/CodeGen/X86/fold-sext-trunc.ll @@ -1,5 +1,9 @@ ; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s ; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -stop-after livedebugvalues -o - | FileCheck %s -check-prefix=MIR + +; RUN: llc --try-experimental-debuginfo-iterators < %s -mtriple=x86_64-- | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -O0 -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -stop-after livedebugvalues -o - | FileCheck %s -check-prefix=MIR + ; PR4050 %0 = type { i64 } diff --git a/llvm/test/CodeGen/X86/fold-zext-trunc.ll b/llvm/test/CodeGen/X86/fold-zext-trunc.ll index 88713d0b553a..7a70b75a028a 100644 --- a/llvm/test/CodeGen/X86/fold-zext-trunc.ll +++ b/llvm/test/CodeGen/X86/fold-zext-trunc.ll @@ -1,5 +1,9 @@ ; RUN: llc < %s | FileCheck %s -check-prefix=ASM ; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -stop-after livedebugvalues -o - | FileCheck %s -check-prefix=MIR + +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck %s -check-prefix=ASM +; RUN: llc --try-experimental-debuginfo-iterators < %s -O0 -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -stop-after livedebugvalues -o - | FileCheck %s -check-prefix=MIR + ; PR9055 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32" target triple = "i686-pc-linux-gnu" diff --git a/llvm/test/CodeGen/X86/label-heapallocsite.ll b/llvm/test/CodeGen/X86/label-heapallocsite.ll index 87225354d733..78bebc1fd174 100644 --- a/llvm/test/CodeGen/X86/label-heapallocsite.ll +++ b/llvm/test/CodeGen/X86/label-heapallocsite.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s | FileCheck --check-prefixes=CHECK %s ; RUN: llc -O0 < %s | FileCheck --check-prefixes=CHECK %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck --check-prefixes=CHECK %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 < %s | FileCheck --check-prefixes=CHECK %s + ; Source to regenerate: ; $ clang -cc1 -triple x86_64-windows-msvc t.cpp -debug-info-kind=limited \ ; -gcodeview -O2 -fms-extensions -emit-llvm -o t.ll diff --git a/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll b/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll index c5b3c74653db..dc582b9c8746 100644 --- a/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll +++ b/llvm/test/CodeGen/X86/machine-outliner-disubprogram.ll @@ -3,6 +3,7 @@ ; Also make sure that the DISubprograms reference the generated unit. ; make sure that if there are two outlined functions in the program, ; RUN: llc %s -verify-machineinstrs -enable-machine-outliner -mtriple=x86_64-apple-darwin -o /dev/null -print-after=machine-outliner +; RUN: llc --try-experimental-debuginfo-iterators %s -verify-machineinstrs -enable-machine-outliner -mtriple=x86_64-apple-darwin -o /dev/null -print-after=machine-outliner define void @f6() #0 !dbg !8 { entry: %dog = alloca i32, align 4 diff --git a/llvm/test/CodeGen/X86/pr53243-tail-call-fastisel.ll b/llvm/test/CodeGen/X86/pr53243-tail-call-fastisel.ll index 333eff8fb008..f21e8df95a4f 100644 --- a/llvm/test/CodeGen/X86/pr53243-tail-call-fastisel.ll +++ b/llvm/test/CodeGen/X86/pr53243-tail-call-fastisel.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -O0 -fast-isel -mtriple=x86_64-- < %s | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 -fast-isel -mtriple=x86_64-- < %s | FileCheck %s define void @test() { ; CHECK-LABEL: test: diff --git a/llvm/test/CodeGen/X86/sink-local-value.ll b/llvm/test/CodeGen/X86/sink-local-value.ll index 4732fb48a922..0e956b00b984 100644 --- a/llvm/test/CodeGen/X86/sink-local-value.ll +++ b/llvm/test/CodeGen/X86/sink-local-value.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 < %s | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 < %s | FileCheck %s target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" target triple = "i386-linux-gnu" diff --git a/llvm/test/DebugInfo/AArch64/cfi-eof-prologue.ll b/llvm/test/DebugInfo/AArch64/cfi-eof-prologue.ll index 0ee313d88866..a7c229456169 100644 --- a/llvm/test/DebugInfo/AArch64/cfi-eof-prologue.ll +++ b/llvm/test/DebugInfo/AArch64/cfi-eof-prologue.ll @@ -15,6 +15,7 @@ ; of CFI instructions. ; RUN: llc -fast-isel -O0 -filetype=asm < %s | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -fast-isel -O0 -filetype=asm < %s | FileCheck %s ; ModuleID = 'test1.cpp' target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" diff --git a/llvm/test/DebugInfo/AArch64/frameindices.ll b/llvm/test/DebugInfo/AArch64/frameindices.ll index 8f736a07034c..d0080afa9085 100644 --- a/llvm/test/DebugInfo/AArch64/frameindices.ll +++ b/llvm/test/DebugInfo/AArch64/frameindices.ll @@ -1,4 +1,5 @@ ; RUN: llc -frame-pointer=all -O0 -fast-isel -filetype=obj < %s | llvm-dwarfdump -v - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -frame-pointer=all -O0 -fast-isel -filetype=obj < %s | llvm-dwarfdump -v - | FileCheck %s ; Test that a variable with multiple entries in the MMI table makes it into the ; debug info. ; diff --git a/llvm/test/DebugInfo/ARM/split-complex.ll b/llvm/test/DebugInfo/ARM/split-complex.ll index 8ac5d4f0aaba..64de07ea61d6 100644 --- a/llvm/test/DebugInfo/ARM/split-complex.ll +++ b/llvm/test/DebugInfo/ARM/split-complex.ll @@ -1,5 +1,8 @@ -; RUN: llc -mtriple=thumbv7-apple-unknown-macho -O0 -filetype=obj -o %t.o %s -; RUN: llvm-dwarfdump -v -debug-info %t.o | FileCheck %s +; RUN: llc -mtriple=thumbv7-apple-unknown-macho -O0 -filetype=obj -o - %s \ +; RUN: | llvm-dwarfdump -v -debug-info - | FileCheck %s + +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=thumbv7-apple-unknown-macho -O0 -filetype=obj -o - %s \ +; RUN: | llvm-dwarfdump -v -debug-info - | FileCheck %s ; generated from (-triple thumbv7-apple-unknown-macho -Os): ; void f(_Complex double c) { c = 0; } diff --git a/llvm/test/DebugInfo/COFF/class-options-common.ll b/llvm/test/DebugInfo/COFF/class-options-common.ll index 903824722b0c..0c6e3a844b0f 100644 --- a/llvm/test/DebugInfo/COFF/class-options-common.ll +++ b/llvm/test/DebugInfo/COFF/class-options-common.ll @@ -2,6 +2,11 @@ ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | FileCheck %s --check-prefix=ASM-INLINE-COMMENTS ; +; Same as above, with experimental debuginfo iterators. +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=ASM-INLINE-COMMENTS +; ; Command to generate function-options.ll ; $ clang++ class-options-common.cpp -S -emit-llvm -g -gcodeview -o class-options-common.ll diff --git a/llvm/test/DebugInfo/COFF/cpp-mangling.ll b/llvm/test/DebugInfo/COFF/cpp-mangling.ll index 09a0d92b610c..beb2180957d7 100644 --- a/llvm/test/DebugInfo/COFF/cpp-mangling.ll +++ b/llvm/test/DebugInfo/COFF/cpp-mangling.ll @@ -1,6 +1,9 @@ ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 -filetype=obj < %s \ ; RUN: | llvm-readobj --codeview - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 -filetype=obj < %s \ +; RUN: | llvm-readobj --codeview - | FileCheck %s + ; C++ source to regenerate: ; namespace foo { ; int bar(int x) { return x * 2; } @@ -111,4 +114,4 @@ attributes #1 = { nounwind readnone } !29 = !{null, !30, !10} !30 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !25, size: 32, flags: DIFlagArtificial | DIFlagObjectPointer) !31 = distinct !DISubprogram(name: "operator<<", linkageName: "??6S@@QAEXH@Z", scope: !25, file: !1, line: 8, type: !28, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, declaration: !27, retainedNodes: !2) -!32 = !DILocation(line: 8, column: 27, scope: !31) \ No newline at end of file +!32 = !DILocation(line: 8, column: 27, scope: !31) diff --git a/llvm/test/DebugInfo/COFF/enum-co.ll b/llvm/test/DebugInfo/COFF/enum-co.ll index 4ccfaae511fc..6ac7445595e8 100644 --- a/llvm/test/DebugInfo/COFF/enum-co.ll +++ b/llvm/test/DebugInfo/COFF/enum-co.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s + ; Command to generate enum-co.ll ; $ clang++ enum-co.cpp -S -emit-llvm -g -gcodeview -o enum-co.ll ; diff --git a/llvm/test/DebugInfo/COFF/function-options.ll b/llvm/test/DebugInfo/COFF/function-options.ll index 863994be1d5e..79658509398f 100644 --- a/llvm/test/DebugInfo/COFF/function-options.ll +++ b/llvm/test/DebugInfo/COFF/function-options.ll @@ -1,5 +1,8 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s + +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s ; ; Command to generate function-options.ll ; $ clang++ function-options.cpp -S -emit-llvm -g -gcodeview -o function-options.ll diff --git a/llvm/test/DebugInfo/COFF/global_visibility.ll b/llvm/test/DebugInfo/COFF/global_visibility.ll index 8da374d3a075..191c2d935cfd 100644 --- a/llvm/test/DebugInfo/COFF/global_visibility.ll +++ b/llvm/test/DebugInfo/COFF/global_visibility.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; ; This test verifies global variables are emitted within the correct scope. ; diff --git a/llvm/test/DebugInfo/COFF/globals.ll b/llvm/test/DebugInfo/COFF/globals.ll index 2724b99dc14b..9113f5fe87f8 100644 --- a/llvm/test/DebugInfo/COFF/globals.ll +++ b/llvm/test/DebugInfo/COFF/globals.ll @@ -5,6 +5,14 @@ ; RUN: llc < %s -filetype=obj -o %t ; RUN: obj2yaml < %t | FileCheck %s --check-prefixes=YAML,YAML-FILE +; Repeat with experimental debuginfo iterators: +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=ASM +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s --check-prefix=OBJ +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s --check-prefix=OBJ +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | obj2yaml | FileCheck %s --check-prefixes=YAML,YAML-STDOUT +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj -o %t +; RUN: obj2yaml < %t | FileCheck %s --check-prefixes=YAML,YAML-FILE + ; C++ source to regenerate: ; $ cat a.cpp ; int first; diff --git a/llvm/test/DebugInfo/COFF/lambda.ll b/llvm/test/DebugInfo/COFF/lambda.ll index 7b575a7b07aa..a171c85ba3c2 100644 --- a/llvm/test/DebugInfo/COFF/lambda.ll +++ b/llvm/test/DebugInfo/COFF/lambda.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | llvm-mc -filetype=obj --triple=i686-windows | llvm-readobj - --codeview | FileCheck %s ; +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=i686-windows | llvm-readobj - --codeview | FileCheck %s +; ; Verify lambda routines are emitted properly in CodeView. ; ; The original source code: diff --git a/llvm/test/DebugInfo/COFF/lines-bb-start.ll b/llvm/test/DebugInfo/COFF/lines-bb-start.ll index 70c96b368c9a..c8693b6f2d57 100644 --- a/llvm/test/DebugInfo/COFF/lines-bb-start.ll +++ b/llvm/test/DebugInfo/COFF/lines-bb-start.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 < %s | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 < %s | FileCheck %s source_filename = "t.c" target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" diff --git a/llvm/test/DebugInfo/COFF/nrvo.ll b/llvm/test/DebugInfo/COFF/nrvo.ll index 7bd9d35b7a00..52b93ddf6bc3 100644 --- a/llvm/test/DebugInfo/COFF/nrvo.ll +++ b/llvm/test/DebugInfo/COFF/nrvo.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s | FileCheck %s --check-prefix=ASM ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s --check-prefix=OBJ +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=ASM +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s --check-prefix=OBJ + ; C++ source to regenerate: ; struct Foo { ; Foo() = default; diff --git a/llvm/test/DebugInfo/COFF/numeric-leaves.ll b/llvm/test/DebugInfo/COFF/numeric-leaves.ll index 2b3f7ed495c6..68f6edc76e27 100644 --- a/llvm/test/DebugInfo/COFF/numeric-leaves.ll +++ b/llvm/test/DebugInfo/COFF/numeric-leaves.ll @@ -2,6 +2,11 @@ ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | FileCheck %s --check-prefix=ASM +; Repeat with experimental debuginfo iterators. +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=ASM + ; C++ source to regenerate: ; $ cat numeric-leaves.cpp ; const long long Minus1 = -1; diff --git a/llvm/test/DebugInfo/COFF/parent-type-scopes.ll b/llvm/test/DebugInfo/COFF/parent-type-scopes.ll index 95c7a5589522..7d91e6538e31 100644 --- a/llvm/test/DebugInfo/COFF/parent-type-scopes.ll +++ b/llvm/test/DebugInfo/COFF/parent-type-scopes.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s -filetype=obj -o %t.o ; RUN: llvm-pdbutil dump -types %t.o | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj -o %t.o +; RUN: llvm-pdbutil dump -types %t.o | FileCheck %s + ; C++ source: ; // Note that MSVC doesn't emit anything about WrapTypedef or WrapTypedef::Inner! ; struct WrapTypedef { diff --git a/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll b/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll index 8118e0ca2ae8..64f97c106a4a 100644 --- a/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll +++ b/llvm/test/DebugInfo/COFF/purge-typedef-udts.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s source_filename = "test/DebugInfo/COFF/purge-typedef-udts.ll" target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" target triple = "i686-pc-windows-msvc19.11.25506" diff --git a/llvm/test/DebugInfo/COFF/thunk.ll b/llvm/test/DebugInfo/COFF/thunk.ll index 39abed580780..d6c15f595358 100644 --- a/llvm/test/DebugInfo/COFF/thunk.ll +++ b/llvm/test/DebugInfo/COFF/thunk.ll @@ -1,7 +1,12 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | FileCheck %s --check-prefix=ASM ; RUN: opt -S -debugger-tune=lldb %s | FileCheck -check-prefix=OPT %s -; + +; Do the same for experimental debuginfo iterators. +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck %s --check-prefix=ASM +; RUN: opt --try-experimental-debuginfo-iterators -S -debugger-tune=lldb %s | FileCheck -check-prefix=OPT %s + ; -- "thunk.cpp" begin -------------------------------------------------------- ; class A { public: virtual bool MyMethod() { return true; } }; ; class B { public: virtual bool MyMethod() { return true; } }; diff --git a/llvm/test/DebugInfo/COFF/type-quals.ll b/llvm/test/DebugInfo/COFF/type-quals.ll index c5953d384d31..461266f72bdc 100644 --- a/llvm/test/DebugInfo/COFF/type-quals.ll +++ b/llvm/test/DebugInfo/COFF/type-quals.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s + ; C++ source to regenerate: ; $ cat m.cpp ; union Union { diff --git a/llvm/test/DebugInfo/COFF/types-cvarargs.ll b/llvm/test/DebugInfo/COFF/types-cvarargs.ll index 68c7a7740b71..bd41d68669f6 100644 --- a/llvm/test/DebugInfo/COFF/types-cvarargs.ll +++ b/llvm/test/DebugInfo/COFF/types-cvarargs.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s + ; C++ source to regenerate: ; $ cat t.cpp ; typedef void (*FuncTypedef)(int, float, ...); diff --git a/llvm/test/DebugInfo/COFF/types-integer-old.ll b/llvm/test/DebugInfo/COFF/types-integer-old.ll index 2ed36662a804..c1c18bca8ed4 100644 --- a/llvm/test/DebugInfo/COFF/types-integer-old.ll +++ b/llvm/test/DebugInfo/COFF/types-integer-old.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; Tests that CodeView integer types are generated even when using Clang's old integer type names. diff --git a/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll b/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll index b7114b87ae5e..6f9987c33ab9 100644 --- a/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll +++ b/llvm/test/DebugInfo/COFF/types-method-ref-qualifiers.ll @@ -1,6 +1,9 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s ; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj - --codeview | FileCheck %s + ; C++ source to regenerate: ; struct A { ; int NoRefQual(); diff --git a/llvm/test/DebugInfo/Generic/2010-10-01-crash.ll b/llvm/test/DebugInfo/Generic/2010-10-01-crash.ll index 8f8d67173c1c..54c4c07f8c23 100644 --- a/llvm/test/DebugInfo/Generic/2010-10-01-crash.ll +++ b/llvm/test/DebugInfo/Generic/2010-10-01-crash.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 %s -o /dev/null +; RUN: llc --try-experimental-debuginfo-iterators -O0 %s -o /dev/null define void @CGRectStandardize(ptr sret(i32) %agg.result, ptr byval(i32) %rect) nounwind ssp !dbg !0 { entry: diff --git a/llvm/test/DebugInfo/Generic/PR20038.ll b/llvm/test/DebugInfo/Generic/PR20038.ll index 024a6abf0591..d7fa11efab5d 100644 --- a/llvm/test/DebugInfo/Generic/PR20038.ll +++ b/llvm/test/DebugInfo/Generic/PR20038.ll @@ -2,6 +2,7 @@ ; XFAIL: target=sparc{{.*}} ; RUN: %llc_dwarf -O0 -filetype=obj -dwarf-linkage-names=All < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not=DW_TAG +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj -dwarf-linkage-names=All < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not=DW_TAG ; IR generated from clang -O0 with: ; struct C { diff --git a/llvm/test/DebugInfo/Generic/dead-argument-order.ll b/llvm/test/DebugInfo/Generic/dead-argument-order.ll index f6cd8092a48d..b2037b0590f2 100644 --- a/llvm/test/DebugInfo/Generic/dead-argument-order.ll +++ b/llvm/test/DebugInfo/Generic/dead-argument-order.ll @@ -1,4 +1,5 @@ ; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck %s +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck %s ; Built from the following source with clang -O1 ; struct S { int i; }; diff --git a/llvm/test/DebugInfo/Generic/discriminated-union.ll b/llvm/test/DebugInfo/Generic/discriminated-union.ll index 0acf478f653a..d267d9b029e9 100644 --- a/llvm/test/DebugInfo/Generic/discriminated-union.ll +++ b/llvm/test/DebugInfo/Generic/discriminated-union.ll @@ -1,6 +1,9 @@ ; RUN: %llc_dwarf -O0 -filetype=obj < %s > %t ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj < %s > %t +; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s + ; Check for a variant part that has two members, one of which has a ; discriminant value. diff --git a/llvm/test/DebugInfo/Generic/disubrange_vla.ll b/llvm/test/DebugInfo/Generic/disubrange_vla.ll index 18dd128ee01f..6554cf46b0be 100644 --- a/llvm/test/DebugInfo/Generic/disubrange_vla.ll +++ b/llvm/test/DebugInfo/Generic/disubrange_vla.ll @@ -4,6 +4,13 @@ ; RUN: llvm-dwarfdump -verbose %t >> %t2 ; RUN: cat %t2 | FileCheck %s +; Repeat the test with experimental debuginfo iterators. +; RUN: rm -rf %t %t2 +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj < %s > %t +; RUN: llvm-dwarfdump -name=vla_expr %t > %t2 +; RUN: llvm-dwarfdump -verbose %t >> %t2 +; RUN: cat %t2 | FileCheck %s + ; This test runs llvm-dwarfdump twice: ; - First to get the debug entry for 'vla_expr'. ; - Second to check that this is the entry referenced in DW_AT_count. diff --git a/llvm/test/DebugInfo/Generic/enum-types.ll b/llvm/test/DebugInfo/Generic/enum-types.ll index 8af9b21bdaa4..288513468b06 100644 --- a/llvm/test/DebugInfo/Generic/enum-types.ll +++ b/llvm/test/DebugInfo/Generic/enum-types.ll @@ -1,5 +1,6 @@ ; ; RUN: %llc_dwarf -filetype=obj -O0 -dwarf-linkage-names=All < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -filetype=obj -O0 -dwarf-linkage-names=All < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s ; Make sure we can handle enums with the same identifier but in enum types of ; different compile units. diff --git a/llvm/test/DebugInfo/Generic/enum.ll b/llvm/test/DebugInfo/Generic/enum.ll index 63665e355edf..11d30d35f7cf 100644 --- a/llvm/test/DebugInfo/Generic/enum.ll +++ b/llvm/test/DebugInfo/Generic/enum.ll @@ -1,6 +1,9 @@ ; RUN: %llc_dwarf -O0 -filetype=obj < %s > %t ; RUN: llvm-dwarfdump -v %t | FileCheck %s +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj < %s > %t +; RUN: llvm-dwarfdump -v %t | FileCheck %s + ; IR generated from the following code compiled with clang -g: ; enum e1 { I, J = 0xffffffffU, K = 0xf000000000000000ULL } a; ; enum e2 { X }; diff --git a/llvm/test/DebugInfo/Generic/import-inlined-declaration.ll b/llvm/test/DebugInfo/Generic/import-inlined-declaration.ll index dc57127dae18..6870cfa81100 100644 --- a/llvm/test/DebugInfo/Generic/import-inlined-declaration.ll +++ b/llvm/test/DebugInfo/Generic/import-inlined-declaration.ll @@ -1,5 +1,7 @@ ; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s + ; namespace ns { ; inline __attribute__((always_inline)) ; void foo() { int a = 4; } diff --git a/llvm/test/DebugInfo/Generic/inlined-vars.ll b/llvm/test/DebugInfo/Generic/inlined-vars.ll index 8a71f09ca01b..f2ef69b47401 100644 --- a/llvm/test/DebugInfo/Generic/inlined-vars.ll +++ b/llvm/test/DebugInfo/Generic/inlined-vars.ll @@ -1,5 +1,7 @@ ; RUN: %llc_dwarf -O0 < %s | FileCheck %s -check-prefix ARGUMENT ; RUN: %llc_dwarf -O0 < %s | FileCheck %s -check-prefix VARIABLE +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 < %s | FileCheck %s -check-prefix ARGUMENT +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 < %s | FileCheck %s -check-prefix VARIABLE ; PR 13202 define i32 @main() uwtable !dbg !5 { diff --git a/llvm/test/DebugInfo/Generic/recursive_inlining.ll b/llvm/test/DebugInfo/Generic/recursive_inlining.ll index 908e408401f8..4636a4d2b722 100644 --- a/llvm/test/DebugInfo/Generic/recursive_inlining.ll +++ b/llvm/test/DebugInfo/Generic/recursive_inlining.ll @@ -1,4 +1,5 @@ ; RUN: %llc_dwarf -filetype=obj -O0 < %s | llvm-dwarfdump -debug-info - | FileCheck %s +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -filetype=obj -O0 < %s | llvm-dwarfdump -debug-info - | FileCheck %s ; This isn't a very pretty test case - I imagine there might be other ways to ; tickle the optimizers into producing the desired code, but I haven't found diff --git a/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll b/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll index 628c2f8b1dec..9514e6fa4db4 100644 --- a/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll +++ b/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll @@ -1,6 +1,9 @@ ; RUN: %llc_dwarf -O0 -filetype=obj < %s > %t ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -O0 -filetype=obj < %s > %t +; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s + ; Check for a univariant discriminated union -- that is, a variant ; part without a discriminant and with just a single variant. diff --git a/llvm/test/DebugInfo/Mips/delay-slot.ll b/llvm/test/DebugInfo/Mips/delay-slot.ll index ed2769f33f4b..f966ac7c1ec5 100644 --- a/llvm/test/DebugInfo/Mips/delay-slot.ll +++ b/llvm/test/DebugInfo/Mips/delay-slot.ll @@ -1,4 +1,5 @@ ; RUN: llc -filetype=obj -O0 -relocation-model=pic < %s -mtriple mips-unknown-linux-gnu | llvm-dwarfdump -a - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -filetype=obj -O0 -relocation-model=pic < %s -mtriple mips-unknown-linux-gnu | llvm-dwarfdump -a - | FileCheck %s ; PR19815 ; Generated using clang -target mips-linux-gnu -g test.c -S -o - -flto|opt -passes=sroa -S diff --git a/llvm/test/DebugInfo/X86/aligned_stack_var.ll b/llvm/test/DebugInfo/X86/aligned_stack_var.ll index 089b3ffab569..f8b5d4c1bdf5 100644 --- a/llvm/test/DebugInfo/X86/aligned_stack_var.ll +++ b/llvm/test/DebugInfo/X86/aligned_stack_var.ll @@ -1,6 +1,9 @@ ; RUN: llc %s -mtriple=x86_64-pc-linux-gnu -O0 -filetype=obj -o %t ; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators %s -mtriple=x86_64-pc-linux-gnu -O0 -filetype=obj -o %t +; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s + ; If stack is realigned, we shouldn't describe locations of local ; variables by giving offset from the frame pointer (%rbp): ; push %rpb diff --git a/llvm/test/DebugInfo/X86/arguments.ll b/llvm/test/DebugInfo/X86/arguments.ll index 767c1385e77f..229a69696ce8 100644 --- a/llvm/test/DebugInfo/X86/arguments.ll +++ b/llvm/test/DebugInfo/X86/arguments.ll @@ -2,6 +2,9 @@ ; RUN: llc -mtriple=x86_64-unknown-unknown -O0 -filetype=obj < %s > %t ; RUN: llvm-dwarfdump %t | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-unknown-unknown -O0 -filetype=obj < %s > %t +; RUN: llvm-dwarfdump %t | FileCheck %s + ; IR generated from clang -g with the following source: ; struct foo { ; foo(const foo&); diff --git a/llvm/test/DebugInfo/X86/asan_debug_info.ll b/llvm/test/DebugInfo/X86/asan_debug_info.ll index 4a4743ee0df5..342e3c86f689 100644 --- a/llvm/test/DebugInfo/X86/asan_debug_info.ll +++ b/llvm/test/DebugInfo/X86/asan_debug_info.ll @@ -3,7 +3,7 @@ ; RUN: llvm-dwarfdump - | FileCheck %s ; RUN: opt --try-experimental-debuginfo-iterators < %s -passes=asan -asan-use-after-return=never -S | \ -; RUN: llc -O0 -filetype=obj - -o - | \ +; RUN: llc --try-experimental-debuginfo-iterators -O0 -filetype=obj - -o - | \ ; RUN: llvm-dwarfdump - | FileCheck %s ; For this test case, ASan used to produce IR which resulted in the following diff --git a/llvm/test/DebugInfo/X86/byvalstruct.ll b/llvm/test/DebugInfo/X86/byvalstruct.ll index d84e4753516b..a23a49d3145d 100644 --- a/llvm/test/DebugInfo/X86/byvalstruct.ll +++ b/llvm/test/DebugInfo/X86/byvalstruct.ll @@ -1,5 +1,7 @@ ; RUN: llc -mtriple=x86_64-apple-macosx10.8.0 -O0 -filetype=obj -o %t %s ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-apple-macosx10.8.0 -O0 -filetype=obj -o %t %s +; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s ; Test that we generate debug info for by-value struct args that are not used. ; ; CHECK: DW_TAG_formal_parameter diff --git a/llvm/test/DebugInfo/X86/convert-linked.ll b/llvm/test/DebugInfo/X86/convert-linked.ll index 6a74a6310bc3..9a0d65b4ccd1 100644 --- a/llvm/test/DebugInfo/X86/convert-linked.ll +++ b/llvm/test/DebugInfo/X86/convert-linked.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple=x86_64 -filetype=obj -O0 < %s | llvm-dwarfdump - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64 -filetype=obj -O0 < %s | llvm-dwarfdump - | FileCheck %s ; CHECK: DW_TAG_compile_unit ; CHECK: [[CU0BT0:0x[0-9a-f]+]]: DW_TAG_base_type diff --git a/llvm/test/DebugInfo/X86/dbg-declare-arg.ll b/llvm/test/DebugInfo/X86/dbg-declare-arg.ll index 8e5c117d16be..c206ab660d22 100644 --- a/llvm/test/DebugInfo/X86/dbg-declare-arg.ll +++ b/llvm/test/DebugInfo/X86/dbg-declare-arg.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -fast-isel=true -filetype=obj -o - %s | llvm-dwarfdump -v - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 -fast-isel=true -filetype=obj -o - %s | llvm-dwarfdump -v - | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "x86_64-apple-macosx10.6.7" diff --git a/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll b/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll index 1de1d0b17b39..2afbf3d8255b 100644 --- a/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll +++ b/llvm/test/DebugInfo/X86/dbg-declare-inalloca.ll @@ -2,6 +2,11 @@ ; RUN: llc < %s | FileCheck %s ; RUN: llc -filetype=obj -O0 < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ +; Same checks, this time with experimental debuginfo iterators. +; RUN: llc --try-experimental-debuginfo-iterators -O0 < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DEBUG +; RUN: llc --try-experimental-debuginfo-iterators < %s | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -filetype=obj -O0 < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ + ; IR generated by the following source: ; struct NonTrivial { ; NonTrivial();// : x(42) {} diff --git a/llvm/test/DebugInfo/X86/dbg-declare.ll b/llvm/test/DebugInfo/X86/dbg-declare.ll index f07ddbb639ec..4cf93c6be0ce 100644 --- a/llvm/test/DebugInfo/X86/dbg-declare.ll +++ b/llvm/test/DebugInfo/X86/dbg-declare.ll @@ -1,6 +1,11 @@ ; RUN: llc < %s -O0 -mtriple x86_64-apple-darwin | FileCheck %s ; RUN: llc < %s -O0 -mtriple x86_64-apple-darwin -filetype=obj \ ; RUN: | llvm-dwarfdump -v - --debug-info | FileCheck %s --check-prefix=DWARF + +; RUN: llc --try-experimental-debuginfo-iterators < %s -O0 -mtriple x86_64-apple-darwin | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators < %s -O0 -mtriple x86_64-apple-darwin -filetype=obj \ +; RUN: | llvm-dwarfdump -v - --debug-info | FileCheck %s --check-prefix=DWARF + ; ; CHECK-LABEL: _foo: diff --git a/llvm/test/DebugInfo/X86/dbg_value_direct.ll b/llvm/test/DebugInfo/X86/dbg_value_direct.ll index 4ac25414f2a7..a1bdd8eab090 100644 --- a/llvm/test/DebugInfo/X86/dbg_value_direct.ll +++ b/llvm/test/DebugInfo/X86/dbg_value_direct.ll @@ -1,4 +1,5 @@ ; RUN: llc -filetype=obj -O0 < %s +; RUN: llc --try-experimental-debuginfo-iterators -filetype=obj -O0 < %s ; Test that we handle DBG_VALUEs in a register without crashing. ; ; Generated from clang with -fsanitize=address: diff --git a/llvm/test/DebugInfo/X86/debug-info-template-parameter.ll b/llvm/test/DebugInfo/X86/debug-info-template-parameter.ll index 10f453e1f21f..e552bb02b03c 100644 --- a/llvm/test/DebugInfo/X86/debug-info-template-parameter.ll +++ b/llvm/test/DebugInfo/X86/debug-info-template-parameter.ll @@ -1,4 +1,5 @@ ; RUN: llc %s -filetype=obj -o - | llvm-dwarfdump -v - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators %s -filetype=obj -o - | llvm-dwarfdump -v - | FileCheck %s ; C++ source to regenerate: diff --git a/llvm/test/DebugInfo/X86/debug_value_list_selectiondag.ll b/llvm/test/DebugInfo/X86/debug_value_list_selectiondag.ll index c4b7b497fb7f..cd21b78487a7 100644 --- a/llvm/test/DebugInfo/X86/debug_value_list_selectiondag.ll +++ b/llvm/test/DebugInfo/X86/debug_value_list_selectiondag.ll @@ -2,6 +2,12 @@ ; RUN: llc %s -fast-isel=true -start-after=codegenprepare -stop-before=finalize-isel -experimental-debug-variable-locations=false -o - | FileCheck --check-prefixes=CHECK,FAST %s ; RUN: llc %s -global-isel=true -start-after=codegenprepare -stop-before=finalize-isel -experimental-debug-variable-locations=false -o - | FileCheck --check-prefixes=CHECK,GLOBAL %s +;; Run with experimental debuginfo iterators. +; RUN: llc --try-experimental-debuginfo-iterators %s -start-after=codegenprepare -stop-before=finalize-isel -experimental-debug-variable-locations=false -o - | FileCheck --check-prefixes=CHECK,DAG %s +; RUN: llc --try-experimental-debuginfo-iterators %s -fast-isel=true -start-after=codegenprepare -stop-before=finalize-isel -experimental-debug-variable-locations=false -o - | FileCheck --check-prefixes=CHECK,FAST %s +;; FIXME: Disabled until global isel support is added. +; run: llc --try-experimental-debuginfo-iterators %s -global-isel=true -start-after=codegenprepare -stop-before=finalize-isel -experimental-debug-variable-locations=false -o - | FileCheck --check-prefixes=CHECK,GLOBAL %s + ;; Run with assignment tracking enabled (use sed to add the module flag). ; RUN: sed 's/;Uncomment-with-sed//g' < %s \ ; RUN: | llc -global-isel=false -start-after=codegenprepare -stop-before=finalize-isel -experimental-debug-variable-locations=false -o - \ diff --git a/llvm/test/DebugInfo/X86/double-declare.ll b/llvm/test/DebugInfo/X86/double-declare.ll index 6f273b142f99..5e422e171ae3 100644 --- a/llvm/test/DebugInfo/X86/double-declare.ll +++ b/llvm/test/DebugInfo/X86/double-declare.ll @@ -1,4 +1,5 @@ ; RUN: llc -mtriple=x86_64-apple-darwin -O0 -filetype=obj -o - < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-apple-darwin -O0 -filetype=obj -o - < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s ; PR33157. Don't crash on duplicate dbg.declare. ; CHECK: DW_TAG_formal_parameter ; CHECK: DW_AT_location [DW_FORM_exprloc] diff --git a/llvm/test/DebugInfo/X86/fi-piece.ll b/llvm/test/DebugInfo/X86/fi-piece.ll index 30144d787555..e7b8afea9059 100644 --- a/llvm/test/DebugInfo/X86/fi-piece.ll +++ b/llvm/test/DebugInfo/X86/fi-piece.ll @@ -1,4 +1,5 @@ ; RUN: llc %s -filetype=obj -o - | llvm-dwarfdump -v - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators %s -filetype=obj -o - | llvm-dwarfdump -v - | FileCheck %s ; Test that multi-DW_OP_piece expressions are emitted for FI variables. ; ; CHECK: .debug_info contents: diff --git a/llvm/test/DebugInfo/X86/implicit_value-double.ll b/llvm/test/DebugInfo/X86/implicit_value-double.ll index 03f7a018ebc4..e0d8e791839c 100644 --- a/llvm/test/DebugInfo/X86/implicit_value-double.ll +++ b/llvm/test/DebugInfo/X86/implicit_value-double.ll @@ -2,7 +2,10 @@ ;; for double type. ; RUN: llc -O0 -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s --check-prefixes=CHECK,BOTH +; RUN: llc --try-experimental-debuginfo-iterators -O0 -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s --check-prefixes=CHECK,BOTH + ; RUN: llc -O0 -debugger-tune=lldb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s --check-prefixes=CHECK,BOTH +; RUN: llc --try-experimental-debuginfo-iterators -O0 -debugger-tune=lldb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s --check-prefixes=CHECK,BOTH ; CHECK: .debug_info contents: ; CHECK: DW_TAG_variable @@ -11,6 +14,7 @@ ; CHECK-NEXT: DW_AT_name ("d") ; RUN: llc -O0 -debugger-tune=sce -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s -check-prefixes=SCE-CHECK,BOTH +; RUN: llc --try-experimental-debuginfo-iterators -O0 -debugger-tune=sce -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s -check-prefixes=SCE-CHECK,BOTH ; SCE-CHECK: .debug_info contents: ; SCE-CHECK: DW_TAG_variable diff --git a/llvm/test/DebugInfo/X86/instr-ref-opt-levels.ll b/llvm/test/DebugInfo/X86/instr-ref-opt-levels.ll index 23eaacac6a74..9313751a5c13 100644 --- a/llvm/test/DebugInfo/X86/instr-ref-opt-levels.ll +++ b/llvm/test/DebugInfo/X86/instr-ref-opt-levels.ll @@ -8,6 +8,17 @@ ; RUN: -fast-isel -experimental-debug-variable-locations \ ; RUN: | FileCheck %s --check-prefix=FASTISEL --implicit-check-not=DBG_VALUE +;; Repeat the tests using experimental debuginfo iterators. +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-- %s -o - -O0 -stop-before=finalize-isel \ +; RUN: -experimental-debug-variable-locations \ +; RUN: | FileCheck %s --check-prefix=O0 --implicit-check-not=DBG_INSTR_REF +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-- %s -o - -O2 -stop-before=finalize-isel \ +; RUN: -experimental-debug-variable-locations \ +; RUN: | FileCheck %s --check-prefix=O2 --implicit-check-not=DBG_VALUE +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-- %s -o - -stop-before=finalize-isel \ +; RUN: -fast-isel -experimental-debug-variable-locations \ +; RUN: | FileCheck %s --check-prefix=FASTISEL --implicit-check-not=DBG_VALUE + ; Test that instruction-referencing variable locations are issued at -O2, but ; normal DBG_VALUEs are issued at -O0. This behaviour is desired as the former ; is slow when applied to unoptimized code. diff --git a/llvm/test/DebugInfo/X86/instr-ref-selectiondag.ll b/llvm/test/DebugInfo/X86/instr-ref-selectiondag.ll index dbbef2b39587..29f7d9353e0a 100644 --- a/llvm/test/DebugInfo/X86/instr-ref-selectiondag.ll +++ b/llvm/test/DebugInfo/X86/instr-ref-selectiondag.ll @@ -12,6 +12,21 @@ ; RUN: | FileCheck %s --check-prefix=FASTISEL-INSTRREF \ ; RUN: --implicit-check-not=DBG_VALUE +;; Repeat tests using experimental debuginfo iterators. +; RUN: llc --try-experimental-debuginfo-iterators %s -mtriple=x86_64-unknown-unknown -o - -stop-before=finalize-isel -experimental-debug-variable-locations=false \ +; RUN: | FileCheck %s --check-prefix=NORMAL \ +; RUN: --implicit-check-not=debug-instr-number \ +; RUN: --implicit-check-not=DBG_INSTR_REF +; RUN: llc --try-experimental-debuginfo-iterators %s -mtriple=x86_64-unknown-unknown -o - -stop-before=finalize-isel \ +; RUN: -experimental-debug-variable-locations -verify-machineinstrs \ +; RUN: | FileCheck %s --check-prefix=INSTRREF \ +; RUN: --implicit-check-not=DBG_VALUE +; RUN: llc --try-experimental-debuginfo-iterators %s -mtriple=x86_64-unknown-unknown -o - -stop-before=finalize-isel \ +; RUN: -experimental-debug-variable-locations -verify-machineinstrs \ +; RUN: -fast-isel \ +; RUN: | FileCheck %s --check-prefix=FASTISEL-INSTRREF \ +; RUN: --implicit-check-not=DBG_VALUE + ; NORMAL: ![[SOCKS:[0-9]+]] = !DILocalVariable(name: "socks", ; NORMAL: ![[KNEES:[0-9]+]] = !DILocalVariable(name: "knees", ; INSTRREF: ![[SOCKS:[0-9]+]] = !DILocalVariable(name: "socks", diff --git a/llvm/test/DebugInfo/X86/missing-abstract-variable.ll b/llvm/test/DebugInfo/X86/missing-abstract-variable.ll index 175a306bdaf6..8d493e593b8b 100644 --- a/llvm/test/DebugInfo/X86/missing-abstract-variable.ll +++ b/llvm/test/DebugInfo/X86/missing-abstract-variable.ll @@ -1,5 +1,5 @@ ; RUN: %llc_dwarf -mtriple x86_64-gnu-linux -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck %s - +; RUN: %llc_dwarf --try-experimental-debuginfo-iterators -mtriple x86_64-gnu-linux -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck %s ; Build from the following source with clang -O2. ; The important details are that 'x's abstract definition is first built during diff --git a/llvm/test/DebugInfo/X86/parameters.ll b/llvm/test/DebugInfo/X86/parameters.ll index 5254fadecbd7..cacdd571c91d 100644 --- a/llvm/test/DebugInfo/X86/parameters.ll +++ b/llvm/test/DebugInfo/X86/parameters.ll @@ -1,5 +1,6 @@ ; ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -O0 -filetype=obj %s -o - | llvm-dwarfdump -v -debug-info - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-unknown-linux-gnu -O0 -filetype=obj %s -o - | llvm-dwarfdump -v -debug-info - | FileCheck %s ; Test case derived from compiling the following source with clang -g: ; diff --git a/llvm/test/DebugInfo/X86/pieces-1.ll b/llvm/test/DebugInfo/X86/pieces-1.ll index f614f78c6581..e0fcd083c7e8 100644 --- a/llvm/test/DebugInfo/X86/pieces-1.ll +++ b/llvm/test/DebugInfo/X86/pieces-1.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 %s -filetype=obj -o - | llvm-dwarfdump -debug-loc - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 %s -filetype=obj -o - | llvm-dwarfdump -debug-loc - | FileCheck %s ; ; rdar://problem/15928306 ; diff --git a/llvm/test/DebugInfo/X86/reference-argument.ll b/llvm/test/DebugInfo/X86/reference-argument.ll index 8747da5071a8..803947ffeef6 100644 --- a/llvm/test/DebugInfo/X86/reference-argument.ll +++ b/llvm/test/DebugInfo/X86/reference-argument.ll @@ -1,5 +1,7 @@ ; RUN: llc -mtriple=x86_64-apple-macosx10.9.0 -filetype=obj -O0 < %s \ ; RUN: | llvm-dwarfdump -v -debug-info - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-apple-macosx10.9.0 -filetype=obj -O0 < %s \ +; RUN: | llvm-dwarfdump -v -debug-info - | FileCheck %s ; ModuleID = 'aggregate-indirect-arg.cpp' ; extracted from debuginfo-tests/aggregate-indirect-arg.cpp diff --git a/llvm/test/DebugInfo/X86/spill-indirect-nrvo.ll b/llvm/test/DebugInfo/X86/spill-indirect-nrvo.ll index b9a504b9b7b6..4bcb08d675a4 100644 --- a/llvm/test/DebugInfo/X86/spill-indirect-nrvo.ll +++ b/llvm/test/DebugInfo/X86/spill-indirect-nrvo.ll @@ -4,11 +4,9 @@ ; RUN: llc -O0 < %s -experimental-debug-variable-locations=true | FileCheck -check-prefixes=CHECK,OPTNONE %s ; RUN: llc --try-experimental-debuginfo-iterators < %s -experimental-debug-variable-locations=false | FileCheck -check-prefixes=CHECK,OPT %s -;; FIXME: RemoveDIs - enable when FastISel support is added. -; run: llc --try-experimental-debuginfo-iterators -O0 < %s -experimental-debug-variable-locations=false | FileCheck -check-prefixes=CHECK,OPTNONE %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 < %s -experimental-debug-variable-locations=false | FileCheck -check-prefixes=CHECK,OPTNONE %s ; RUN: llc --try-experimental-debuginfo-iterators < %s -experimental-debug-variable-locations=true | FileCheck -check-prefixes=CHECK,OPT %s -;; FIXME: RemoveDIs - enable when FastISel support is added. -; run: llc --try-experimental-debuginfo-iterators -O0 < %s -experimental-debug-variable-locations=true | FileCheck -check-prefixes=CHECK,OPTNONE %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 < %s -experimental-debug-variable-locations=true | FileCheck -check-prefixes=CHECK,OPTNONE %s ; Make sure we insert DW_OP_deref when spilling indirect DBG_VALUE instructions. diff --git a/llvm/test/DebugInfo/X86/sret.ll b/llvm/test/DebugInfo/X86/sret.ll index f06b097d07b9..222fb05b6df8 100644 --- a/llvm/test/DebugInfo/X86/sret.ll +++ b/llvm/test/DebugInfo/X86/sret.ll @@ -1,8 +1,7 @@ ; RUN: llc -split-dwarf-file=foo.dwo -O0 %s -mtriple=x86_64-unknown-linux-gnu -filetype=obj -o %t ; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s --check-prefix=CHECK-DWO -;; FIXME: RemoveDIs - enable when FastISel support is added. -; run: llc --try-experimental-debuginfo-iterators -split-dwarf-file=foo.dwo -O0 %s -mtriple=x86_64-unknown-linux-gnu -filetype=obj -o %t -; run: llvm-dwarfdump -debug-info %t | FileCheck %s --check-prefix=CHECK-DWO +; RUN: llc --try-experimental-debuginfo-iterators -split-dwarf-file=foo.dwo -O0 %s -mtriple=x86_64-unknown-linux-gnu -filetype=obj -o %t +; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s --check-prefix=CHECK-DWO ; Based on the debuginfo-tests/sret.cpp code. @@ -12,8 +11,7 @@ ; RUN: llc -O0 -fast-isel=true -mtriple=x86_64-apple-darwin -filetype=obj -o - %s | llvm-dwarfdump -debug-info - | FileCheck -check-prefixes=CHECK,FASTISEL %s ; RUN: llc -O0 -fast-isel=false -mtriple=x86_64-apple-darwin -filetype=obj -o - %s | llvm-dwarfdump -debug-info - | FileCheck -check-prefixes=CHECK,SDAG %s -;; FIXME: RemoveDIs - enable when FastISel support is added. -; run: llc --try-experimental-debuginfo-iterators -O0 -fast-isel=true -mtriple=x86_64-apple-darwin -filetype=obj -o - %s | llvm-dwarfdump -debug-info - | FileCheck -check-prefixes=CHECK,FASTISEL %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 -fast-isel=true -mtriple=x86_64-apple-darwin -filetype=obj -o - %s | llvm-dwarfdump -debug-info - | FileCheck -check-prefixes=CHECK,FASTISEL %s ; RUN: llc --try-experimental-debuginfo-iterators -O0 -fast-isel=false -mtriple=x86_64-apple-darwin -filetype=obj -o - %s | llvm-dwarfdump -debug-info - | FileCheck -check-prefixes=CHECK,SDAG %s ; CHECK: _ZN1B9AInstanceEv diff --git a/llvm/test/DebugInfo/X86/subreg.ll b/llvm/test/DebugInfo/X86/subreg.ll index 671af9e05fe2..f826cc5a9c93 100644 --- a/llvm/test/DebugInfo/X86/subreg.ll +++ b/llvm/test/DebugInfo/X86/subreg.ll @@ -1,4 +1,5 @@ ; RUN: llc %s -mtriple=x86_64-pc-linux-gnu -O0 -o - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators %s -mtriple=x86_64-pc-linux-gnu -O0 -o - | FileCheck %s ; We are testing that a value in a 16 bit register gets reported as ; being in its superregister. diff --git a/llvm/test/DebugInfo/X86/subregisters.ll b/llvm/test/DebugInfo/X86/subregisters.ll index 4e279aba3d8b..5dc5a8033f1e 100644 --- a/llvm/test/DebugInfo/X86/subregisters.ll +++ b/llvm/test/DebugInfo/X86/subregisters.ll @@ -1,5 +1,8 @@ ; RUN: llc -mtriple=x86_64-apple-darwin %s -o %t.o -filetype=obj -O0 ; RUN: llvm-dwarfdump -v %t.o | FileCheck %s + +; RUN: llc --try-experimental-debuginfo-iterators -mtriple=x86_64-apple-darwin %s -o %t.o -filetype=obj -O0 +; RUN: llvm-dwarfdump -v %t.o | FileCheck %s ; ; Test that on x86_64, the 32-bit subregister esi is emitted as ; subregister of the 64-bit rsi. diff --git a/llvm/test/DebugInfo/X86/vla.ll b/llvm/test/DebugInfo/X86/vla.ll index 26322ec0fdd7..f90cf2aadc71 100644 --- a/llvm/test/DebugInfo/X86/vla.ll +++ b/llvm/test/DebugInfo/X86/vla.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=x86_64-apple-darwin -filetype=asm %s -o - | FileCheck %s +; RUN: llc --try-experimental-debuginfo-iterators -O0 -mtriple=x86_64-apple-darwin -filetype=asm %s -o - | FileCheck %s ; Ensure that we generate an indirect location for the variable length array a. ; CHECK: ##DEBUG_VALUE: vla:a <- [DW_OP_deref] [{{\$r[a-z]+}}+0] ; CHECK: DW_OP_breg{{[0-9]}} -- GitLab From fc18b13492880ba8597333c6050a18ec6db0f831 Mon Sep 17 00:00:00 2001 From: Boian Petkantchin Date: Fri, 5 Jan 2024 07:14:07 -0800 Subject: [PATCH 124/728] [mlir][mesh] In sharding attr use FlatSymbolRefAttr instead of SymbolRefAttr (#76886) Analogous to func.call use FlatSymbolRefAttr to reference the corresponding mesh. --- mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td | 8 ++++---- mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td | 4 ++-- .../mlir/Dialect/Mesh/Interfaces/ShardingInterface.h | 4 ++-- mlir/lib/Dialect/Mesh/IR/MeshOps.cpp | 2 +- mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp | 2 +- mlir/test/Dialect/Mesh/invalid.mlir | 8 ++++++++ 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td index 060d54b82efa..bda6467e9c5d 100644 --- a/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td +++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td @@ -79,7 +79,7 @@ def MeshSharding : AttrDef { let mnemonic = "shard"; let parameters = (ins - AttrParameter<"::mlir::SymbolRefAttr", "cluster placed">:$cluster, + AttrParameter<"::mlir::FlatSymbolRefAttr", "cluster placed">:$cluster, ArrayRefParameter<"MeshAxesAttr">:$split_axes, OptionalArrayRefParameter<"MeshAxis">:$partial_axes, OptionalParameter<"::mlir::mesh::Partial">:$partial_type @@ -91,7 +91,7 @@ def MeshSharding : AttrDef { The MeshSharding attribute could be used in the encoding of a `RankedTensorType` or the mesh.shard op. it contains three sub-attributes: - 1. `cluster`: this attribute is a SymbolRefAttr that refers to the mesh + 1. `cluster`: this attribute is a FlatSymbolRefAttr that refers to the mesh cluster where the distributed tensor is placed. The symbol must resolve to a `mesh.cluster` operation. @@ -145,7 +145,7 @@ def MeshSharding : AttrDef { }]; let builders = [ - AttrBuilder<(ins "SymbolRefAttr":$cluster, + AttrBuilder<(ins "FlatSymbolRefAttr":$cluster, "ArrayRef>":$split_axes, "ArrayRef": $partial_axes, "mesh::Partial": $partial_type), [{ @@ -156,7 +156,7 @@ def MeshSharding : AttrDef { return $_get($_ctxt, cluster, splitAxesAttr, partial_axes, partial_type); }]>, - AttrBuilder<(ins "SymbolRefAttr":$cluster, + AttrBuilder<(ins "FlatSymbolRefAttr":$cluster, "ArrayRef>":$split_axes), [{ return MeshShardingAttr::get($_ctxt, cluster, split_axes, {}, Partial::Sum); }]> diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td index 1934bdfb4270..f459077ea120 100644 --- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td +++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td @@ -196,12 +196,12 @@ def Mesh_ShardOp : Mesh_Op<"shard", [Pure, SameOperandsAndResultType]> { ``` }]; let arguments = (ins - Builtin_RankedTensor:$src, + AnyRankedTensor:$src, MeshSharding:$shard, UnitAttr:$annotate_for_users ); let results = (outs - Builtin_RankedTensor:$result + AnyRankedTensor:$result ); let assemblyFormat = [{ $src `to` $shard (`annotate_for_users` $annotate_for_users^)? attr-dict `:` diff --git a/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterface.h b/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterface.h index 201c0151754e..a32274d857f1 100644 --- a/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterface.h +++ b/mlir/include/mlir/Dialect/Mesh/Interfaces/ShardingInterface.h @@ -25,13 +25,13 @@ struct ShardingOption { // An array of int array. The sub-array at the i-th position signifies the // mesh axes the i-th loop will be sharded on. ShardingArray shardingArray = {}; - SymbolRefAttr cluster = nullptr; + FlatSymbolRefAttr cluster = nullptr; // `empty` being true indicates that no sharding information can be inferred // at present. Note that it is different from the case where an operation is // not sharded. bool empty = false; ShardingOption() = default; - ShardingOption(ShardingArray shardingArray, SymbolRefAttr cluster) + ShardingOption(ShardingArray shardingArray, FlatSymbolRefAttr cluster) : shardingArray(std::move(shardingArray)), cluster(cluster) {} }; diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp index c3d8f1d45610..6667d409df8b 100644 --- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp +++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp @@ -266,7 +266,7 @@ void ClusterShapeOp::build(OpBuilder &odsBuilder, OperationState &odsState, LogicalResult MeshShardingAttr::verify(function_ref emitError, - SymbolRefAttr, ArrayRef splitAxes, + FlatSymbolRefAttr, ArrayRef splitAxes, ArrayRef partialAxes, Partial) { // TODO: At present cluster symbol ref is not verified. This is due to the // difficulty in fetching the corresponding symbol op based on an attribute. diff --git a/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp b/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp index ee885ab16b7b..dca7e86e6f07 100644 --- a/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp +++ b/mlir/lib/Dialect/Mesh/Interfaces/ShardingInterface.cpp @@ -215,7 +215,7 @@ namespace { // Update the given `shardingOption` according to `meshAxes` and `loopIdx` static LogicalResult fillShardingOption(Operation *op, ShardingOption &shardingOption, - SymbolRefAttr cluster, + FlatSymbolRefAttr cluster, ArrayRef meshAxes, unsigned loopIdx) { if ((shardingOption.cluster && cluster && diff --git a/mlir/test/Dialect/Mesh/invalid.mlir b/mlir/test/Dialect/Mesh/invalid.mlir index 3ee578a37235..3e1b04da0dfd 100644 --- a/mlir/test/Dialect/Mesh/invalid.mlir +++ b/mlir/test/Dialect/Mesh/invalid.mlir @@ -70,6 +70,14 @@ func.func @mesh_axis_negtive_in_partial( // ----- +func.func @sharding_attribute_invalid_nested_symbol(%arg0 : tensor<4x8xf32>) { + // expected-error@+2 {{custom op 'mesh.shard' invalid kind of attribute specified}} + // expected-error@+1 {{custom op 'mesh.shard' failed to parse MeshSharding parameter 'cluster' which is to be a `::mlir::FlatSymbolRefAttr`}} + %0 = mesh.shard %arg0 to <@a::@b, [[0]]> : tensor<4x8xf32> +} + +// ----- + mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @cluster_shape_mesh_axis_out_of_bounds() -> (index, index) { -- GitLab From 922b7b8bf465ddc292fa91bd6a860510a1eea6e2 Mon Sep 17 00:00:00 2001 From: "Joel E. Denny" Date: Fri, 5 Jan 2024 10:22:10 -0500 Subject: [PATCH 125/728] [Clang][OpenMP] Fix stdio.h wrapper when glibc includes again (#77017) Since D154036 landed (2a65d0388ca0 on July 7, 2023), I've been seeing many (40-50) libomptarget test failures with errors like the following on some of our test systems: ``` /auto/software/gcc/x86_64/gcc-11.1.0/lib/gcc/x86_64-pc-linux-gnu/11.1.0/../../../../include/c++/11.1.0/cstdio:99:11: error: no member named 'fpos_t' in the global namespace 99 | using ::fpos_t; | ~~^ ``` This patch fixes that and doesn't break our other test sytems. I've looked through the glibc history and at LLVM's libc stdio.h to give me confidence that this solution should work on other systems. Of course, there might be use cases I've overlooked, so feedback is appreciated. --- clang/lib/Headers/llvm_libc_wrappers/stdio.h | 34 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/clang/lib/Headers/llvm_libc_wrappers/stdio.h b/clang/lib/Headers/llvm_libc_wrappers/stdio.h index 0870f3e741ec..950f91b3763e 100644 --- a/clang/lib/Headers/llvm_libc_wrappers/stdio.h +++ b/clang/lib/Headers/llvm_libc_wrappers/stdio.h @@ -6,15 +6,41 @@ // //===----------------------------------------------------------------------===// -#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ -#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ - #if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__) #error "This file is for GPU offloading compilation only" #endif #include_next +// In some old versions of glibc, other standard headers sometimes define +// special macros (e.g., __need_FILE) before including stdio.h to cause stdio.h +// to produce special definitions. Future includes of stdio.h when those +// special macros are undefined are expected to produce the normal definitions +// from stdio.h. +// +// We do not apply our include guard (__CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__) +// unconditionally to the above include_next. Otherwise, after an occurrence of +// the first glibc stdio.h use case described above, the include_next would be +// skipped for remaining includes of stdio.h, leaving required symbols +// undefined. +// +// We make the following assumptions to handle all use cases: +// +// 1. If the above include_next produces special glibc definitions, then (a) it +// does not produce the normal definitions that we must intercept below, (b) +// the current file was included from a glibc header that already defined +// __GLIBC__ (usually by including glibc's ), and (c) the above +// include_next does not define _STDIO_H. In that case, we skip the rest of +// the current file and don't guard against future includes. +// 2. If the above include_next produces the normal stdio.h definitions, then +// either (a) __GLIBC__ is not defined because C headers are from some other +// libc implementation or (b) the above include_next defines _STDIO_H to +// prevent the above include_next from having any effect in the future. +#if !defined(__GLIBC__) || defined(_STDIO_H) + +#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ +#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ + #if __has_include() #if defined(__HIP__) || defined(__CUDA__) @@ -50,3 +76,5 @@ #endif #endif // __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__ + +#endif -- GitLab From de5039545ee4e9f5aa7023632d5c4baa5e782e51 Mon Sep 17 00:00:00 2001 From: Congcong Cai Date: Fri, 5 Jan 2024 23:37:58 +0800 Subject: [PATCH 126/728] [clang]Transform uninstantiated ExceptionSpec in `TemplateInstantiator` (#77073) Fixes: #77071 `SubstituteDeducedTypeTransform` will transform type and it will visit uninstantiated `ExceptionSpecInfo`, which will cause odd behavior. --- clang/docs/ReleaseNotes.rst | 3 ++- clang/include/clang/AST/Type.h | 2 ++ clang/lib/AST/Type.cpp | 7 +++++++ clang/lib/Sema/SemaTemplateDeduction.cpp | 11 +++++++++++ clang/lib/Sema/SemaTemplateInstantiate.cpp | 4 +--- .../SemaCXX/dependent-noexcept-uninstantiated.cpp | 4 +++- 6 files changed, 26 insertions(+), 5 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index f5159cdc8a3b..9b6e00b23121 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -609,7 +609,8 @@ Bug Fixes in This Version - Clang will correctly evaluate ``noexcept`` expression for template functions of template classes. Fixes (`#68543 `_, - `#42496 `_) + `#42496 `_, + `#77071 `_) - Fixed an issue when a shift count larger than ``__INT64_MAX__``, in a right shift operation, could result in missing warnings about ``shift count >= width of type`` or internal compiler error. diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h index 1afa69367286..9e9f896ebef7 100644 --- a/clang/include/clang/AST/Type.h +++ b/clang/include/clang/AST/Type.h @@ -4224,6 +4224,8 @@ public: ExceptionSpecInfo() = default; ExceptionSpecInfo(ExceptionSpecificationType EST) : Type(EST) {} + + void instantiate(); }; /// Extra information about a function prototype. ExtProtoInfo is not diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 160a725939cc..a894d3289eb1 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -3414,6 +3414,13 @@ StringRef FunctionType::getNameForCallConv(CallingConv CC) { llvm_unreachable("Invalid calling convention."); } +void FunctionProtoType::ExceptionSpecInfo::instantiate() { + assert(Type == EST_Uninstantiated); + NoexceptExpr = + cast(SourceTemplate->getType())->getNoexceptExpr(); + Type = EST_DependentNoexcept; +} + FunctionProtoType::FunctionProtoType(QualType result, ArrayRef params, QualType canonical, const ExtProtoInfo &epi) diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp index 699e0985e595..015b0abaf0e5 100644 --- a/clang/lib/Sema/SemaTemplateDeduction.cpp +++ b/clang/lib/Sema/SemaTemplateDeduction.cpp @@ -4737,6 +4737,7 @@ namespace { QualType Replacement; bool ReplacementIsPack; bool UseTypeSugar; + using inherited = TreeTransform; public: SubstituteDeducedTypeTransform(Sema &SemaRef, DependentAuto DA) @@ -4797,6 +4798,16 @@ namespace { // Lambdas never need to be transformed. return E; } + bool TransformExceptionSpec(SourceLocation Loc, + FunctionProtoType::ExceptionSpecInfo &ESI, + SmallVectorImpl &Exceptions, + bool &Changed) { + if (ESI.Type == EST_Uninstantiated) { + ESI.instantiate(); + Changed = true; + } + return inherited::TransformExceptionSpec(Loc, ESI, Exceptions, Changed); + } QualType Apply(TypeLoc TL) { // Create some scratch storage for the transformed type locations. diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp index 27a4b68fd59a..09dd119ed1b9 100644 --- a/clang/lib/Sema/SemaTemplateInstantiate.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp @@ -1639,9 +1639,7 @@ bool TemplateInstantiator::TransformExceptionSpec( SourceLocation Loc, FunctionProtoType::ExceptionSpecInfo &ESI, SmallVectorImpl &Exceptions, bool &Changed) { if (ESI.Type == EST_Uninstantiated) { - ESI.NoexceptExpr = cast(ESI.SourceTemplate->getType()) - ->getNoexceptExpr(); - ESI.Type = EST_DependentNoexcept; + ESI.instantiate(); Changed = true; } return inherited::TransformExceptionSpec(Loc, ESI, Exceptions, Changed); diff --git a/clang/test/SemaCXX/dependent-noexcept-uninstantiated.cpp b/clang/test/SemaCXX/dependent-noexcept-uninstantiated.cpp index ddb269890227..3821f18e7bf2 100644 --- a/clang/test/SemaCXX/dependent-noexcept-uninstantiated.cpp +++ b/clang/test/SemaCXX/dependent-noexcept-uninstantiated.cpp @@ -6,6 +6,8 @@ using B = char; template struct C { template void f0() noexcept(sizeof(T) == sizeof(A) && sizeof(V) == sizeof(B)) {} + template auto f1(V a) noexcept(1) {return a;} }; -void (C::*tmp1)() noexcept = &C::f0; +void (C::*tmp0)() noexcept = &C::f0; +int (C::*tmp1)(int) noexcept = &C::f1; -- GitLab From 63ab56e2a3a05bcb1ca367d9c40eaabd18cb42d5 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 5 Jan 2024 16:35:46 +0100 Subject: [PATCH 127/728] [Clang] Regenerate test checks (NFC) --- clang/test/CodeGen/RISCV/riscv-abi.cpp | 2 +- clang/test/CodeGen/isfpclass.c | 26 +- clang/test/CodeGenCXX/nrvo.cpp | 844 +++++++++--------- .../master_taskloop_in_reduction_codegen.cpp | 58 +- ...ter_taskloop_simd_in_reduction_codegen.cpp | 58 +- .../test/OpenMP/task_in_reduction_codegen.cpp | 48 +- .../OpenMP/taskloop_in_reduction_codegen.cpp | 58 +- .../taskloop_simd_in_reduction_codegen.cpp | 58 +- 8 files changed, 576 insertions(+), 576 deletions(-) diff --git a/clang/test/CodeGen/RISCV/riscv-abi.cpp b/clang/test/CodeGen/RISCV/riscv-abi.cpp index b92256206ece..aa18afe41f6d 100644 --- a/clang/test/CodeGen/RISCV/riscv-abi.cpp +++ b/clang/test/CodeGen/RISCV/riscv-abi.cpp @@ -1,4 +1,4 @@ -// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --filter "^define |^entry:" --version 2 +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --filter "^define |^entry:" --version 2 // RUN: %clang_cc1 -triple riscv32 -emit-llvm %s -o - \ // RUN: | FileCheck -check-prefixes=ILP32-ILP32F-ILP32D,ILP32-ILP32F,ILP32 %s // RUN: %clang_cc1 -triple riscv32 -target-feature +f -target-abi ilp32f -emit-llvm %s -o - \ diff --git a/clang/test/CodeGen/isfpclass.c b/clang/test/CodeGen/isfpclass.c index 88d7a21b9733..430f5d94211f 100644 --- a/clang/test/CodeGen/isfpclass.c +++ b/clang/test/CodeGen/isfpclass.c @@ -15,7 +15,7 @@ _Bool check_isfpclass_finite(float x) { // CHECK-LABEL: define dso_local noundef i1 @check_isfpclass_finite_strict // CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 504) #[[ATTR6:[0-9]+]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 504) #[[ATTR5:[0-9]+]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isfpclass_finite_strict(float x) { @@ -24,7 +24,7 @@ _Bool check_isfpclass_finite_strict(float x) { } // CHECK-LABEL: define dso_local noundef i1 @check_isfpclass_nan_f32 -// CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] { +// CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = fcmp uno float [[X]], 0.000000e+00 // CHECK-NEXT: ret i1 [[TMP0]] @@ -36,7 +36,7 @@ _Bool check_isfpclass_nan_f32(float x) { // CHECK-LABEL: define dso_local noundef i1 @check_isfpclass_nan_f32_strict // CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 3) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 3) #[[ATTR5]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isfpclass_nan_f32_strict(float x) { @@ -57,7 +57,7 @@ _Bool check_isfpclass_snan_f64(double x) { // CHECK-LABEL: define dso_local noundef i1 @check_isfpclass_snan_f64_strict // CHECK-SAME: (double noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f64(double [[X]], i32 1) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f64(double [[X]], i32 1) #[[ATTR5]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isfpclass_snan_f64_strict(double x) { @@ -66,7 +66,7 @@ _Bool check_isfpclass_snan_f64_strict(double x) { } // CHECK-LABEL: define dso_local noundef i1 @check_isfpclass_zero_f16 -// CHECK-SAME: (half noundef [[X:%.*]]) local_unnamed_addr #[[ATTR3]] { +// CHECK-SAME: (half noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = fcmp oeq half [[X]], 0xH0000 // CHECK-NEXT: ret i1 [[TMP0]] @@ -78,7 +78,7 @@ _Bool check_isfpclass_zero_f16(_Float16 x) { // CHECK-LABEL: define dso_local noundef i1 @check_isfpclass_zero_f16_strict // CHECK-SAME: (half noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f16(half [[X]], i32 96) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f16(half [[X]], i32 96) #[[ATTR5]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isfpclass_zero_f16_strict(_Float16 x) { @@ -89,7 +89,7 @@ _Bool check_isfpclass_zero_f16_strict(_Float16 x) { // CHECK-LABEL: define dso_local noundef i1 @check_isnan // CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 3) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 3) #[[ATTR5]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isnan(float x) { @@ -100,7 +100,7 @@ _Bool check_isnan(float x) { // CHECK-LABEL: define dso_local noundef i1 @check_isinf // CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 516) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 516) #[[ATTR5]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isinf(float x) { @@ -111,7 +111,7 @@ _Bool check_isinf(float x) { // CHECK-LABEL: define dso_local noundef i1 @check_isfinite // CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 504) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 504) #[[ATTR5]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isfinite(float x) { @@ -122,7 +122,7 @@ _Bool check_isfinite(float x) { // CHECK-LABEL: define dso_local noundef i1 @check_isnormal // CHECK-SAME: (float noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 264) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[X]], i32 264) #[[ATTR5]] // CHECK-NEXT: ret i1 [[TMP0]] // _Bool check_isnormal(float x) { @@ -137,7 +137,7 @@ typedef int __attribute__((ext_vector_type(4))) int4; typedef long __attribute__((ext_vector_type(4))) long4; // CHECK-LABEL: define dso_local noundef <4 x i32> @check_isfpclass_nan_v4f32 -// CHECK-SAME: (<4 x float> noundef [[X:%.*]]) local_unnamed_addr #[[ATTR3]] { +// CHECK-SAME: (<4 x float> noundef [[X:%.*]]) local_unnamed_addr #[[ATTR0]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = fcmp uno <4 x float> [[X]], zeroinitializer // CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP0]] to <4 x i32> @@ -150,7 +150,7 @@ int4 check_isfpclass_nan_v4f32(float4 x) { // CHECK-LABEL: define dso_local noundef <4 x i32> @check_isfpclass_nan_strict_v4f32 // CHECK-SAME: (<4 x float> noundef [[X:%.*]]) local_unnamed_addr #[[ATTR2]] { // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i1> @llvm.is.fpclass.v4f32(<4 x float> [[X]], i32 3) #[[ATTR6]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call <4 x i1> @llvm.is.fpclass.v4f32(<4 x float> [[X]], i32 3) #[[ATTR5]] // CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i1> [[TMP0]] to <4 x i32> // CHECK-NEXT: ret <4 x i32> [[TMP1]] // @@ -160,7 +160,7 @@ int4 check_isfpclass_nan_strict_v4f32(float4 x) { } // CHECK-LABEL: define dso_local void @check_isfpclass_nan_v4f64 -// CHECK-SAME: (ptr noalias nocapture writeonly sret(<4 x i64>) align 16 [[AGG_RESULT:%.*]], ptr nocapture noundef readonly [[TMP0:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] { +// CHECK-SAME: (ptr noalias nocapture writeonly sret(<4 x i64>) align 16 [[AGG_RESULT:%.*]], ptr nocapture noundef readonly [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[X:%.*]] = load <4 x double>, ptr [[TMP0]], align 16, !tbaa [[TBAA2:![0-9]+]] // CHECK-NEXT: [[TMP1:%.*]] = fcmp uno <4 x double> [[X]], zeroinitializer diff --git a/clang/test/CodeGenCXX/nrvo.cpp b/clang/test/CodeGenCXX/nrvo.cpp index d8a88832b4de..975cf6ee1b66 100644 --- a/clang/test/CodeGenCXX/nrvo.cpp +++ b/clang/test/CodeGenCXX/nrvo.cpp @@ -87,8 +87,8 @@ X test0() { // http://wg21.link/p2025r2#ex-2 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -118,8 +118,8 @@ X test0() { // http://wg21.link/p2025r2#ex-2 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -149,8 +149,8 @@ X test0() { // http://wg21.link/p2025r2#ex-2 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -188,8 +188,8 @@ X test1(bool B) { // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: call void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) @@ -218,43 +218,43 @@ X test1(bool B) { // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) // CHECK-EH-03-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD1:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD1:%.*]] // CHECK-EH-03: invoke.cont2: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[EHCLEANUP:%.*]] // CHECK-EH-03: lpad1: -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP6]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP7:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP7]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP5]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP6]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT5:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT5:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: if.end: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD1]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD1]] // CHECK-EH-03: invoke.cont3: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP]] // CHECK-EH-03: cleanup: // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT4:%.*]] unwind label [[LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT4:%.*]] unwind label [[LPAD]] // CHECK-EH-03: invoke.cont4: // CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) // CHECK-EH-03-NEXT: ret void @@ -262,7 +262,7 @@ X test1(bool B) { // CHECK-EH-03-NEXT: br label [[EHCLEANUP]] // CHECK-EH-03: ehcleanup: // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT7:%.*]] unwind label [[TERMINATE_LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT7:%.*]] unwind label [[TERMINATE_LPAD]] // CHECK-EH-03: invoke.cont7: // CHECK-EH-03-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-03: eh.resume: @@ -272,10 +272,10 @@ X test1(bool B) { // CHECK-EH-03-NEXT: [[LPAD_VAL8:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL8]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP8:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP9:%.*]] = extractvalue { ptr, i32 } [[TMP8]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP9]]) #[[ATTR6:[0-9]+]] +// CHECK-EH-03-NEXT: [[TMP7:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP8:%.*]] = extractvalue { ptr, i32 } [[TMP7]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP8]]) #[[ATTR6:[0-9]+]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z5test2b( @@ -292,37 +292,37 @@ X test1(bool B) { // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) // CHECK-EH-11-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD1:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD1:%.*]] // CHECK-EH-11: invoke.cont2: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[EHCLEANUP:%.*]] // CHECK-EH-11: lpad1: -// CHECK-EH-11-NEXT: [[TMP5:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP6]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP7:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP7]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP5]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP6]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EHCLEANUP]] // CHECK-EH-11: if.end: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD1]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD1]] // CHECK-EH-11: invoke.cont3: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP]] @@ -357,8 +357,8 @@ X test2(bool B) { // CHECK-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -394,8 +394,8 @@ X test2(bool B) { // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -431,8 +431,8 @@ X test2(bool B) { // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -481,8 +481,8 @@ extern "C" void exit(int) throw(); // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -500,8 +500,8 @@ extern "C" void exit(int) throw(); // CHECK: nrvo.skipdtor: // CHECK-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-NEXT: ] // CHECK: cleanup.cont: // CHECK-NEXT: call void @exit(i32 noundef 1) #[[ATTR4]] @@ -523,8 +523,8 @@ extern "C" void exit(int) throw(); // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -542,8 +542,8 @@ extern "C" void exit(int) throw(); // CHECK-EH-03: nrvo.skipdtor: // CHECK-EH-03-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-EH-03-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-EH-03-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-EH-03-NEXT: ] // CHECK-EH-03: cleanup.cont: // CHECK-EH-03-NEXT: call void @exit(i32 noundef 1) #[[ATTR7:[0-9]+]] @@ -565,8 +565,8 @@ extern "C" void exit(int) throw(); // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -584,8 +584,8 @@ extern "C" void exit(int) throw(); // CHECK-EH-11: nrvo.skipdtor: // CHECK-EH-11-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-EH-11-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-EH-11-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-EH-11-NEXT: ] // CHECK-EH-11: cleanup.cont: // CHECK-EH-11-NEXT: call void @exit(i32 noundef 1) #[[ATTR6]] @@ -615,59 +615,59 @@ void may_throw(); // CHECK-EH-03-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: invoke void @_Z9may_throwv() -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: br label [[TRY_CONT:%.*]] // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr @_ZTI1X -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr @_ZTI1X +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP1]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP2]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CATCH_DISPATCH:%.*]] // CHECK-EH-03: catch.dispatch: // CHECK-EH-03-NEXT: [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = call i32 @llvm.eh.typeid.for(ptr @_ZTI1X) #[[ATTR7]] -// CHECK-EH-03-NEXT: [[MATCHES:%.*]] = icmp eq i32 [[SEL]], [[TMP4]] +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = call i32 @llvm.eh.typeid.for(ptr @_ZTI1X) #[[ATTR7]] +// CHECK-EH-03-NEXT: [[MATCHES:%.*]] = icmp eq i32 [[SEL]], [[TMP3]] // CHECK-EH-03-NEXT: br i1 [[MATCHES]], label [[CATCH:%.*]], label [[EH_RESUME:%.*]] // CHECK-EH-03: catch: // CHECK-EH-03-NEXT: [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = call ptr @__cxa_get_exception_ptr(ptr [[EXN]]) #[[ATTR7]] -// CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[X]], ptr noundef nonnull align 1 dereferenceable(1) [[TMP5]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = call ptr @__cxa_get_exception_ptr(ptr [[EXN]]) #[[ATTR7]] +// CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[X]], ptr noundef nonnull align 1 dereferenceable(1) [[TMP4]]) +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: invoke.cont1: -// CHECK-EH-03-NEXT: [[TMP7:%.*]] = call ptr @__cxa_begin_catch(ptr [[EXN]]) #[[ATTR7]] +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = call ptr @__cxa_begin_catch(ptr [[EXN]]) #[[ATTR7]] // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD2:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD2:%.*]] // CHECK-EH-03: invoke.cont3: // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT5:%.*]] unwind label [[LPAD4:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT5:%.*]] unwind label [[LPAD4:%.*]] // CHECK-EH-03: lpad2: -// CHECK-EH-03-NEXT: [[TMP8:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP9:%.*]] = extractvalue { ptr, i32 } [[TMP8]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP9]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP10:%.*]] = extractvalue { ptr, i32 } [[TMP8]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP10]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP6:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP7:%.*]] = extractvalue { ptr, i32 } [[TMP6]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP7]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP8:%.*]] = extractvalue { ptr, i32 } [[TMP6]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP8]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT6:%.*]] unwind label [[TERMINATE_LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT6:%.*]] unwind label [[TERMINATE_LPAD]] // CHECK-EH-03: invoke.cont5: // CHECK-EH-03-NEXT: call void @__cxa_end_catch() // CHECK-EH-03-NEXT: ret void // CHECK-EH-03: lpad4: -// CHECK-EH-03-NEXT: [[TMP11:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP12:%.*]] = extractvalue { ptr, i32 } [[TMP11]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP12]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP13:%.*]] = extractvalue { ptr, i32 } [[TMP11]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP13]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP9:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP10:%.*]] = extractvalue { ptr, i32 } [[TMP9]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP10]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP11:%.*]] = extractvalue { ptr, i32 } [[TMP9]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP11]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[EHCLEANUP:%.*]] // CHECK-EH-03: invoke.cont6: // CHECK-EH-03-NEXT: br label [[EHCLEANUP]] // CHECK-EH-03: ehcleanup: // CHECK-EH-03-NEXT: invoke void @__cxa_end_catch() -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT7:%.*]] unwind label [[TERMINATE_LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT7:%.*]] unwind label [[TERMINATE_LPAD]] // CHECK-EH-03: invoke.cont7: // CHECK-EH-03-NEXT: br label [[EH_RESUME]] // CHECK-EH-03: try.cont: @@ -680,10 +680,10 @@ void may_throw(); // CHECK-EH-03-NEXT: [[LPAD_VAL10:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL9]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL10]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP14:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP15:%.*]] = extractvalue { ptr, i32 } [[TMP14]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP15]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP12:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP13:%.*]] = extractvalue { ptr, i32 } [[TMP12]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP13]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z5test5v( @@ -694,45 +694,45 @@ void may_throw(); // CHECK-EH-11-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: invoke void @_Z9may_throwv() -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: br label [[TRY_CONT:%.*]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: catch ptr @_ZTI1X -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: catch ptr @_ZTI1X +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP1]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP2]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CATCH_DISPATCH:%.*]] // CHECK-EH-11: catch.dispatch: // CHECK-EH-11-NEXT: [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = call i32 @llvm.eh.typeid.for(ptr @_ZTI1X) #[[ATTR6]] -// CHECK-EH-11-NEXT: [[MATCHES:%.*]] = icmp eq i32 [[SEL]], [[TMP4]] +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = call i32 @llvm.eh.typeid.for(ptr @_ZTI1X) #[[ATTR6]] +// CHECK-EH-11-NEXT: [[MATCHES:%.*]] = icmp eq i32 [[SEL]], [[TMP3]] // CHECK-EH-11-NEXT: br i1 [[MATCHES]], label [[CATCH:%.*]], label [[EH_RESUME:%.*]] // CHECK-EH-11: catch: // CHECK-EH-11-NEXT: [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP5:%.*]] = call ptr @__cxa_get_exception_ptr(ptr [[EXN]]) #[[ATTR6]] -// CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[X]], ptr noundef nonnull align 1 dereferenceable(1) [[TMP5]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-11-NEXT: [[TMP4:%.*]] = call ptr @__cxa_get_exception_ptr(ptr [[EXN]]) #[[ATTR6]] +// CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[X]], ptr noundef nonnull align 1 dereferenceable(1) [[TMP4]]) +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-11: invoke.cont1: -// CHECK-EH-11-NEXT: [[TMP7:%.*]] = call ptr @__cxa_begin_catch(ptr [[EXN]]) #[[ATTR6]] +// CHECK-EH-11-NEXT: [[TMP5:%.*]] = call ptr @__cxa_begin_catch(ptr [[EXN]]) #[[ATTR6]] // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD2:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD2:%.*]] // CHECK-EH-11: invoke.cont3: // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: call void @__cxa_end_catch() // CHECK-EH-11-NEXT: ret void // CHECK-EH-11: lpad2: -// CHECK-EH-11-NEXT: [[TMP8:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP9:%.*]] = extractvalue { ptr, i32 } [[TMP8]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP9]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP10:%.*]] = extractvalue { ptr, i32 } [[TMP8]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP10]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP6:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP7:%.*]] = extractvalue { ptr, i32 } [[TMP6]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP7]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP8:%.*]] = extractvalue { ptr, i32 } [[TMP6]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP8]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: invoke void @__cxa_end_catch() -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT4:%.*]] unwind label [[TERMINATE_LPAD]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT4:%.*]] unwind label [[TERMINATE_LPAD]] // CHECK-EH-11: invoke.cont4: // CHECK-EH-11-NEXT: br label [[EH_RESUME]] // CHECK-EH-11: try.cont: @@ -745,10 +745,10 @@ void may_throw(); // CHECK-EH-11-NEXT: [[LPAD_VAL7:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL6]], 1 // CHECK-EH-11-NEXT: resume { ptr, i32 } [[LPAD_VAL7]] // CHECK-EH-11: terminate.lpad: -// CHECK-EH-11-NEXT: [[TMP11:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: catch ptr null -// CHECK-EH-11-NEXT: [[TMP12:%.*]] = extractvalue { ptr, i32 } [[TMP11]], 0 -// CHECK-EH-11-NEXT: call void @__clang_call_terminate(ptr [[TMP12]]) #[[ATTR7:[0-9]+]] +// CHECK-EH-11-NEXT: [[TMP9:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: catch ptr null +// CHECK-EH-11-NEXT: [[TMP10:%.*]] = extractvalue { ptr, i32 } [[TMP9]], 0 +// CHECK-EH-11-NEXT: call void @__clang_call_terminate(ptr [[TMP10]]) #[[ATTR7:[0-9]+]] // CHECK-EH-11-NEXT: unreachable // X test5() { // http://wg21.link/p2025r2#ex-14 @@ -779,19 +779,19 @@ X test5() { // http://wg21.link/p2025r2#ex-14 // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[A]]) // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[A]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[A]]) // CHECK-EH-03-NEXT: ret void // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP1]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP2]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[A]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: invoke.cont1: // CHECK-EH-03-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-03: eh.resume: @@ -801,10 +801,10 @@ X test5() { // http://wg21.link/p2025r2#ex-14 // CHECK-EH-03-NEXT: [[LPAD_VAL2:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL2]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP5]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP3]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP4]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z5test6v( @@ -816,17 +816,17 @@ X test5() { // http://wg21.link/p2025r2#ex-14 // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[A]]) // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[A]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[A]]) #[[ATTR6]] // CHECK-EH-11-NEXT: ret void // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP1]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP2]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[A]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: eh.resume: @@ -849,8 +849,8 @@ X test6() { // CHECK-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -877,8 +877,8 @@ X test6() { // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -905,8 +905,8 @@ X test6() { // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -942,8 +942,8 @@ X test7(bool b) { // CHECK-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] // CHECK: if.then: // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -979,8 +979,8 @@ X test7(bool b) { // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -1016,8 +1016,8 @@ X test7(bool b) { // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -1095,8 +1095,8 @@ Y test9() { // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] // CHECK: if.then: // CHECK-NEXT: call void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) @@ -1122,27 +1122,27 @@ Y test9() { // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: if.else: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-03: invoke.cont1: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP]] @@ -1158,10 +1158,10 @@ Y test9() { // CHECK-EH-03-NEXT: [[LPAD_VAL3:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL3]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP6]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP5]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z6test10b( @@ -1176,27 +1176,27 @@ Y test9() { // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: if.else: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-11: invoke.cont1: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP]] @@ -1228,8 +1228,8 @@ X test10(bool b) { // http://wg21.link/p2025r2#ex-3 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1255,27 +1255,27 @@ X test10(bool b) { // http://wg21.link/p2025r2#ex-3 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: if.end: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-03: invoke.cont1: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP]] @@ -1291,10 +1291,10 @@ X test10(bool b) { // http://wg21.link/p2025r2#ex-3 // CHECK-EH-03-NEXT: [[LPAD_VAL3:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL3]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP6]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP5]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z6test11b( @@ -1309,27 +1309,27 @@ X test10(bool b) { // http://wg21.link/p2025r2#ex-3 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: if.end: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-11: invoke.cont1: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP]] @@ -1363,8 +1363,8 @@ X test11(bool b) { // http://wg21.link/p2025r2#ex-5 // CHECK: do.body: // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: store i32 2, ptr [[CLEANUP_DEST_SLOT]], align 4 @@ -1382,8 +1382,8 @@ X test11(bool b) { // http://wg21.link/p2025r2#ex-5 // CHECK: nrvo.skipdtor: // CHECK-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-NEXT: i32 2, label [[DO_END:%.*]] -// CHECK-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-NEXT: i32 2, label [[DO_END:%.*]] +// CHECK-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-NEXT: ] // CHECK: do.end: // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1406,8 +1406,8 @@ X test11(bool b) { // http://wg21.link/p2025r2#ex-5 // CHECK-EH-03: do.body: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i32 2, ptr [[CLEANUP_DEST_SLOT]], align 4 @@ -1425,8 +1425,8 @@ X test11(bool b) { // http://wg21.link/p2025r2#ex-5 // CHECK-EH-03: nrvo.skipdtor: // CHECK-EH-03-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-03-NEXT: i32 2, label [[DO_END:%.*]] -// CHECK-EH-03-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-EH-03-NEXT: i32 2, label [[DO_END:%.*]] +// CHECK-EH-03-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-EH-03-NEXT: ] // CHECK-EH-03: do.end: // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1449,8 +1449,8 @@ X test11(bool b) { // http://wg21.link/p2025r2#ex-5 // CHECK-EH-11: do.body: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i32 2, ptr [[CLEANUP_DEST_SLOT]], align 4 @@ -1468,8 +1468,8 @@ X test11(bool b) { // http://wg21.link/p2025r2#ex-5 // CHECK-EH-11: nrvo.skipdtor: // CHECK-EH-11-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-11-NEXT: i32 2, label [[DO_END:%.*]] -// CHECK-EH-11-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-EH-11-NEXT: i32 2, label [[DO_END:%.*]] +// CHECK-EH-11-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-EH-11-NEXT: ] // CHECK-EH-11: do.end: // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1497,8 +1497,8 @@ X test12(bool b) { // http://wg21.link/p2025r2#ex-6 // CHECK-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1525,8 +1525,8 @@ X test12(bool b) { // http://wg21.link/p2025r2#ex-6 // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1553,8 +1553,8 @@ X test12(bool b) { // http://wg21.link/p2025r2#ex-6 // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1591,8 +1591,8 @@ X test13(bool b) { // http://wg21.link/p2025r2#ex-7 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: call void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) @@ -1627,28 +1627,28 @@ X test13(bool b) { // http://wg21.link/p2025r2#ex-7 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: if.end: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-03: invoke.cont1: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 @@ -1656,7 +1656,7 @@ X test13(bool b) { // http://wg21.link/p2025r2#ex-7 // CHECK-EH-03-NEXT: br i1 [[NRVO_VAL]], label [[NRVO_SKIPDTOR:%.*]], label [[NRVO_UNUSED:%.*]] // CHECK-EH-03: nrvo.unused: // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] // CHECK-EH-03: invoke.cont2: // CHECK-EH-03-NEXT: br label [[NRVO_SKIPDTOR]] // CHECK-EH-03: nrvo.skipdtor: @@ -1673,10 +1673,10 @@ X test13(bool b) { // http://wg21.link/p2025r2#ex-7 // CHECK-EH-03-NEXT: [[LPAD_VAL4:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL4]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP6]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP5]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z6test14b( @@ -1692,28 +1692,28 @@ X test13(bool b) { // http://wg21.link/p2025r2#ex-7 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: if.end: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-11: invoke.cont1: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 @@ -1753,8 +1753,8 @@ X test14(bool b) { // http://wg21.link/p2025r2#ex-8 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: call void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) @@ -1789,28 +1789,28 @@ X test14(bool b) { // http://wg21.link/p2025r2#ex-8 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: if.end: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-03: invoke.cont1: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 @@ -1818,7 +1818,7 @@ X test14(bool b) { // http://wg21.link/p2025r2#ex-8 // CHECK-EH-03-NEXT: br i1 [[NRVO_VAL]], label [[NRVO_SKIPDTOR:%.*]], label [[NRVO_UNUSED:%.*]] // CHECK-EH-03: nrvo.unused: // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] // CHECK-EH-03: invoke.cont2: // CHECK-EH-03-NEXT: br label [[NRVO_SKIPDTOR]] // CHECK-EH-03: nrvo.skipdtor: @@ -1835,10 +1835,10 @@ X test14(bool b) { // http://wg21.link/p2025r2#ex-8 // CHECK-EH-03-NEXT: [[LPAD_VAL4:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL4]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP6]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP5]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z6test15b( @@ -1854,28 +1854,28 @@ X test14(bool b) { // http://wg21.link/p2025r2#ex-8 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: if.end: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: invoke void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[LPAD]] // CHECK-EH-11: invoke.cont1: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 @@ -1916,17 +1916,17 @@ X test15(bool b) { // http://wg21.link/p2025r2#ex-15 // CHECK-EH-11-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 0 // CHECK-EH-11-NEXT: store ptr [[X]], ptr [[TMP0]], align 4 // CHECK-EH-11-NEXT: invoke void @"_ZZ6test16vENK3$_0clEv"(ptr sret([[CLASS_X]]) align 1 [[AGG_TMP]], ptr noundef nonnull align 4 dereferenceable(4) [[REF_TMP]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: invoke void @_Z8ConsumeX1X(ptr noundef [[AGG_TMP]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD1:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD1:%.*]] // CHECK-EH-11: invoke.cont2: // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_TMP]]) #[[ATTR6]] // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: ret void // CHECK-EH-11: lpad: // CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: cleanup // CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 // CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 // CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 @@ -1934,7 +1934,7 @@ X test15(bool b) { // http://wg21.link/p2025r2#ex-15 // CHECK-EH-11-NEXT: br label [[EHCLEANUP:%.*]] // CHECK-EH-11: lpad1: // CHECK-EH-11-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: cleanup // CHECK-EH-11-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 // CHECK-EH-11-NEXT: store ptr [[TMP5]], ptr [[EXN_SLOT]], align 4 // CHECK-EH-11-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 1 @@ -1972,8 +1972,8 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK: if.then: // CHECK-NEXT: br label [[IMPOSSIBLE:%.*]] // CHECK: impossible: -// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 3 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP0]], 3 // CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] // CHECK: if.then1: // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -1985,44 +1985,44 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK: while.body: // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP3:%.*]] = icmp eq i32 [[TMP2]], 0 +// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP3:%.*]] = icmp eq i32 [[TMP1]], 0 // CHECK-NEXT: br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END5:%.*]] // CHECK: if.then4: // CHECK-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: br label [[CLEANUP:%.*]] // CHECK: if.end5: -// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP6:%.*]] = icmp eq i32 [[TMP3]], 1 +// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP6:%.*]] = icmp eq i32 [[TMP2]], 1 // CHECK-NEXT: br i1 [[CMP6]], label [[IF_THEN7:%.*]], label [[IF_END8:%.*]] // CHECK: if.then7: // CHECK-NEXT: store i32 4, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: br label [[CLEANUP]] // CHECK: if.end8: -// CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP9:%.*]] = icmp eq i32 [[TMP4]], 2 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP9:%.*]] = icmp eq i32 [[TMP3]], 2 // CHECK-NEXT: br i1 [[CMP9]], label [[IF_THEN10:%.*]], label [[IF_END11:%.*]] // CHECK: if.then10: // CHECK-NEXT: store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: br label [[CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]] // CHECK: if.end11: -// CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP12:%.*]] = icmp eq i32 [[TMP5]], 3 +// CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP12:%.*]] = icmp eq i32 [[TMP4]], 3 // CHECK-NEXT: br i1 [[CMP12]], label [[IF_THEN13:%.*]], label [[IF_END14:%.*]] // CHECK: if.then13: // CHECK-NEXT: store i32 2, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: br label [[CLEANUP]] // CHECK: if.end14: -// CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP15:%.*]] = icmp eq i32 [[TMP6]], 4 +// CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP15:%.*]] = icmp eq i32 [[TMP5]], 4 // CHECK-NEXT: br i1 [[CMP15]], label [[IF_THEN16:%.*]], label [[IF_END17:%.*]] // CHECK: if.then16: // CHECK-NEXT: call void @exit(i32 noundef 1) #[[ATTR4]] // CHECK-NEXT: br label [[IF_END17]] // CHECK: if.end17: -// CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP18:%.*]] = icmp eq i32 [[TMP7]], 5 +// CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP18:%.*]] = icmp eq i32 [[TMP6]], 5 // CHECK-NEXT: br i1 [[CMP18]], label [[IF_THEN19:%.*]], label [[IF_END20:%.*]] // CHECK: if.then19: // CHECK-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -2040,11 +2040,11 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK: nrvo.skipdtor: // CHECK-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-NEXT: i32 1, label [[RETURN]] -// CHECK-NEXT: i32 4, label [[WHILE_END:%.*]] -// CHECK-NEXT: i32 3, label [[WHILE_BODY]] -// CHECK-NEXT: i32 2, label [[IMPOSSIBLE]] +// CHECK-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-NEXT: i32 1, label [[RETURN]] +// CHECK-NEXT: i32 4, label [[WHILE_END:%.*]] +// CHECK-NEXT: i32 3, label [[WHILE_BODY]] +// CHECK-NEXT: i32 2, label [[IMPOSSIBLE]] // CHECK-NEXT: ] // CHECK: cleanup.cont: // CHECK-NEXT: br label [[WHILE_BODY]], !llvm.loop [[LOOP3]] @@ -2068,8 +2068,8 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: br label [[IMPOSSIBLE:%.*]] // CHECK-EH-03: impossible: -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 3 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP0]], 3 // CHECK-EH-03-NEXT: br i1 [[CMP]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then1: // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -2081,44 +2081,44 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK-EH-03: while.body: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP3:%.*]] = icmp eq i32 [[TMP2]], 0 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP3:%.*]] = icmp eq i32 [[TMP1]], 0 // CHECK-EH-03-NEXT: br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END5:%.*]] // CHECK-EH-03: if.then4: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-03: if.end5: -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP6:%.*]] = icmp eq i32 [[TMP3]], 1 +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP6:%.*]] = icmp eq i32 [[TMP2]], 1 // CHECK-EH-03-NEXT: br i1 [[CMP6]], label [[IF_THEN7:%.*]], label [[IF_END8:%.*]] // CHECK-EH-03: if.then7: // CHECK-EH-03-NEXT: store i32 4, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP]] // CHECK-EH-03: if.end8: -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP9:%.*]] = icmp eq i32 [[TMP4]], 2 +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP9:%.*]] = icmp eq i32 [[TMP3]], 2 // CHECK-EH-03-NEXT: br i1 [[CMP9]], label [[IF_THEN10:%.*]], label [[IF_END11:%.*]] // CHECK-EH-03: if.then10: // CHECK-EH-03-NEXT: store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP]] // CHECK-EH-03: if.end11: -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP12:%.*]] = icmp eq i32 [[TMP5]], 3 +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP12:%.*]] = icmp eq i32 [[TMP4]], 3 // CHECK-EH-03-NEXT: br i1 [[CMP12]], label [[IF_THEN13:%.*]], label [[IF_END14:%.*]] // CHECK-EH-03: if.then13: // CHECK-EH-03-NEXT: store i32 2, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: br label [[CLEANUP]] // CHECK-EH-03: if.end14: -// CHECK-EH-03-NEXT: [[TMP6:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP15:%.*]] = icmp eq i32 [[TMP6]], 4 +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP15:%.*]] = icmp eq i32 [[TMP5]], 4 // CHECK-EH-03-NEXT: br i1 [[CMP15]], label [[IF_THEN16:%.*]], label [[IF_END17:%.*]] // CHECK-EH-03: if.then16: // CHECK-EH-03-NEXT: call void @exit(i32 noundef 1) #[[ATTR7]] // CHECK-EH-03-NEXT: br label [[IF_END17]] // CHECK-EH-03: if.end17: -// CHECK-EH-03-NEXT: [[TMP7:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP18:%.*]] = icmp eq i32 [[TMP7]], 5 +// CHECK-EH-03-NEXT: [[TMP6:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP18:%.*]] = icmp eq i32 [[TMP6]], 5 // CHECK-EH-03-NEXT: br i1 [[CMP18]], label [[IF_THEN19:%.*]], label [[IF_END20:%.*]] // CHECK-EH-03: if.then19: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -2136,11 +2136,11 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK-EH-03: nrvo.skipdtor: // CHECK-EH-03-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-EH-03-NEXT: i32 1, label [[RETURN]] -// CHECK-EH-03-NEXT: i32 4, label [[WHILE_END:%.*]] -// CHECK-EH-03-NEXT: i32 3, label [[WHILE_BODY]] -// CHECK-EH-03-NEXT: i32 2, label [[IMPOSSIBLE]] +// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-EH-03-NEXT: i32 1, label [[RETURN]] +// CHECK-EH-03-NEXT: i32 4, label [[WHILE_END:%.*]] +// CHECK-EH-03-NEXT: i32 3, label [[WHILE_BODY]] +// CHECK-EH-03-NEXT: i32 2, label [[IMPOSSIBLE]] // CHECK-EH-03-NEXT: ] // CHECK-EH-03: cleanup.cont: // CHECK-EH-03-NEXT: br label [[WHILE_BODY]] @@ -2164,8 +2164,8 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: br label [[IMPOSSIBLE:%.*]] // CHECK-EH-11: impossible: -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 3 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP0]], 3 // CHECK-EH-11-NEXT: br i1 [[CMP]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then1: // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) @@ -2177,44 +2177,44 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK-EH-11: while.body: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP3:%.*]] = icmp eq i32 [[TMP2]], 0 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP3:%.*]] = icmp eq i32 [[TMP1]], 0 // CHECK-EH-11-NEXT: br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END5:%.*]] // CHECK-EH-11: if.then4: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: store i32 1, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP:%.*]] // CHECK-EH-11: if.end5: -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP6:%.*]] = icmp eq i32 [[TMP3]], 1 +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP6:%.*]] = icmp eq i32 [[TMP2]], 1 // CHECK-EH-11-NEXT: br i1 [[CMP6]], label [[IF_THEN7:%.*]], label [[IF_END8:%.*]] // CHECK-EH-11: if.then7: // CHECK-EH-11-NEXT: store i32 4, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP]] // CHECK-EH-11: if.end8: -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP9:%.*]] = icmp eq i32 [[TMP4]], 2 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP9:%.*]] = icmp eq i32 [[TMP3]], 2 // CHECK-EH-11-NEXT: br i1 [[CMP9]], label [[IF_THEN10:%.*]], label [[IF_END11:%.*]] // CHECK-EH-11: if.then10: // CHECK-EH-11-NEXT: store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]] // CHECK-EH-11: if.end11: -// CHECK-EH-11-NEXT: [[TMP5:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP12:%.*]] = icmp eq i32 [[TMP5]], 3 +// CHECK-EH-11-NEXT: [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP12:%.*]] = icmp eq i32 [[TMP4]], 3 // CHECK-EH-11-NEXT: br i1 [[CMP12]], label [[IF_THEN13:%.*]], label [[IF_END14:%.*]] // CHECK-EH-11: if.then13: // CHECK-EH-11-NEXT: store i32 2, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: br label [[CLEANUP]] // CHECK-EH-11: if.end14: -// CHECK-EH-11-NEXT: [[TMP6:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP15:%.*]] = icmp eq i32 [[TMP6]], 4 +// CHECK-EH-11-NEXT: [[TMP5:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP15:%.*]] = icmp eq i32 [[TMP5]], 4 // CHECK-EH-11-NEXT: br i1 [[CMP15]], label [[IF_THEN16:%.*]], label [[IF_END17:%.*]] // CHECK-EH-11: if.then16: // CHECK-EH-11-NEXT: call void @exit(i32 noundef 1) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[IF_END17]] // CHECK-EH-11: if.end17: -// CHECK-EH-11-NEXT: [[TMP7:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP18:%.*]] = icmp eq i32 [[TMP7]], 5 +// CHECK-EH-11-NEXT: [[TMP6:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP18:%.*]] = icmp eq i32 [[TMP6]], 5 // CHECK-EH-11-NEXT: br i1 [[CMP18]], label [[IF_THEN19:%.*]], label [[IF_END20:%.*]] // CHECK-EH-11: if.then19: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -2232,11 +2232,11 @@ void test16() { // http://wg21.link/p2025r2#ex-9 // CHECK-EH-11: nrvo.skipdtor: // CHECK-EH-11-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-EH-11-NEXT: i32 1, label [[RETURN]] -// CHECK-EH-11-NEXT: i32 4, label [[WHILE_END:%.*]] -// CHECK-EH-11-NEXT: i32 3, label [[WHILE_BODY]] -// CHECK-EH-11-NEXT: i32 2, label [[IMPOSSIBLE]] +// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-EH-11-NEXT: i32 1, label [[RETURN]] +// CHECK-EH-11-NEXT: i32 4, label [[WHILE_END:%.*]] +// CHECK-EH-11-NEXT: i32 3, label [[WHILE_BODY]] +// CHECK-EH-11-NEXT: i32 2, label [[IMPOSSIBLE]] // CHECK-EH-11-NEXT: ] // CHECK-EH-11: cleanup.cont: // CHECK-EH-11-NEXT: br label [[WHILE_BODY]], !llvm.loop [[LOOP3]] @@ -2285,8 +2285,8 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK-NEXT: store i32 [[I:%.*]], ptr [[I_ADDR]], align 4 // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP0]], 0 // CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -2304,14 +2304,14 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK: nrvo.skipdtor: // CHECK-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-NEXT: ] // CHECK: cleanup.cont: // CHECK-NEXT: store i1 false, ptr [[NRVO1]], align 1 // CHECK-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP2]], 1 +// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP1]], 1 // CHECK-NEXT: br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_END4:%.*]] // CHECK: if.then3: // CHECK-NEXT: store i1 true, ptr [[NRVO1]], align 1 @@ -2329,8 +2329,8 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK: nrvo.skipdtor8: // CHECK-NEXT: [[CLEANUP_DEST9:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-NEXT: switch i32 [[CLEANUP_DEST9]], label [[UNREACHABLE]] [ -// CHECK-NEXT: i32 0, label [[CLEANUP_CONT10:%.*]] -// CHECK-NEXT: i32 1, label [[RETURN]] +// CHECK-NEXT: i32 0, label [[CLEANUP_CONT10:%.*]] +// CHECK-NEXT: i32 1, label [[RETURN]] // CHECK-NEXT: ] // CHECK: cleanup.cont10: // CHECK-NEXT: store i1 false, ptr [[NRVO11]], align 1 @@ -2361,8 +2361,8 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK-EH-03-NEXT: store i32 [[I:%.*]], ptr [[I_ADDR]], align 4 // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP0]], 0 // CHECK-EH-03-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -2380,14 +2380,14 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK-EH-03: nrvo.skipdtor: // CHECK-EH-03-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-EH-03-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-EH-03-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-EH-03-NEXT: ] // CHECK-EH-03: cleanup.cont: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO1]], align 1 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-03-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP2]], 1 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-03-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP1]], 1 // CHECK-EH-03-NEXT: br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_END4:%.*]] // CHECK-EH-03: if.then3: // CHECK-EH-03-NEXT: store i1 true, ptr [[NRVO1]], align 1 @@ -2405,8 +2405,8 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK-EH-03: nrvo.skipdtor8: // CHECK-EH-03-NEXT: [[CLEANUP_DEST9:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-03-NEXT: switch i32 [[CLEANUP_DEST9]], label [[UNREACHABLE]] [ -// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT10:%.*]] -// CHECK-EH-03-NEXT: i32 1, label [[RETURN]] +// CHECK-EH-03-NEXT: i32 0, label [[CLEANUP_CONT10:%.*]] +// CHECK-EH-03-NEXT: i32 1, label [[RETURN]] // CHECK-EH-03-NEXT: ] // CHECK-EH-03: cleanup.cont10: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO11]], align 1 @@ -2437,8 +2437,8 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK-EH-11-NEXT: store i32 [[I:%.*]], ptr [[I_ADDR]], align 4 // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP:%.*]] = icmp eq i32 [[TMP0]], 0 // CHECK-EH-11-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 @@ -2456,14 +2456,14 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK-EH-11: nrvo.skipdtor: // CHECK-EH-11-NEXT: [[CLEANUP_DEST:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: switch i32 [[CLEANUP_DEST]], label [[UNREACHABLE:%.*]] [ -// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] -// CHECK-EH-11-NEXT: i32 1, label [[RETURN:%.*]] +// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT:%.*]] +// CHECK-EH-11-NEXT: i32 1, label [[RETURN:%.*]] // CHECK-EH-11-NEXT: ] // CHECK-EH-11: cleanup.cont: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO1]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = load i32, ptr [[I_ADDR]], align 4 -// CHECK-EH-11-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP2]], 1 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i32, ptr [[I_ADDR]], align 4 +// CHECK-EH-11-NEXT: [[CMP2:%.*]] = icmp eq i32 [[TMP1]], 1 // CHECK-EH-11-NEXT: br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_END4:%.*]] // CHECK-EH-11: if.then3: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO1]], align 1 @@ -2481,8 +2481,8 @@ X test17(int i) { // http://wg21.link/p2025r2#ex-10 // CHECK-EH-11: nrvo.skipdtor8: // CHECK-EH-11-NEXT: [[CLEANUP_DEST9:%.*]] = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4 // CHECK-EH-11-NEXT: switch i32 [[CLEANUP_DEST9]], label [[UNREACHABLE]] [ -// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT10:%.*]] -// CHECK-EH-11-NEXT: i32 1, label [[RETURN]] +// CHECK-EH-11-NEXT: i32 0, label [[CLEANUP_CONT10:%.*]] +// CHECK-EH-11-NEXT: i32 1, label [[RETURN]] // CHECK-EH-11-NEXT: ] // CHECK-EH-11: cleanup.cont10: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO11]], align 1 @@ -2528,22 +2528,22 @@ X test18(int i) { // http://wg21.link/p2025r2#ex-11 // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0 -// CHECK-EH-11-NEXT: store ptr [[AGG_RESULT]], ptr [[TMP1]], align 4 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0 +// CHECK-EH-11-NEXT: store ptr [[AGG_RESULT]], ptr [[TMP0]], align 4 // CHECK-EH-11-NEXT: invoke void @"_ZZ6test19vENK3$_0clEv"(ptr sret([[CLASS_X]]) align 1 [[L]], ptr noundef nonnull align 4 dereferenceable(4) [[REF_TMP]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: store i1 true, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[L]]) #[[ATTR6]] // CHECK-EH-11-NEXT: [[NRVO_VAL:%.*]] = load i1, ptr [[NRVO]], align 1 // CHECK-EH-11-NEXT: br i1 [[NRVO_VAL]], label [[NRVO_SKIPDTOR:%.*]], label [[NRVO_UNUSED:%.*]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: nrvo.unused: @@ -2668,19 +2668,19 @@ const volatile X test21() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERVKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) // CHECK-EH-03-NEXT: ret void // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP1]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP2]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: invoke.cont1: // CHECK-EH-03-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-03: eh.resume: @@ -2690,10 +2690,10 @@ const volatile X test21() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-03-NEXT: [[LPAD_VAL2:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL2]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP5]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP3]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP4]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z6test22v( @@ -2705,17 +2705,17 @@ const volatile X test21() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERVKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: ret void // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP1]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP0]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP2]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: eh.resume: @@ -2739,8 +2739,8 @@ X test22() { // http://wg21.link/p2025r2#ex-19 // CHECK-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: // CHECK-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -2772,8 +2772,8 @@ X test22() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-03-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-03-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-03-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-03-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-03-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-03: if.then: // CHECK-EH-03-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -2789,19 +2789,19 @@ X test22() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-03: if.end: // CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) // CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERVKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-03: invoke.cont: // CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) // CHECK-EH-03-NEXT: br label [[RETURN]] // CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-03-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-03-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-03-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-03-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-03-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] // CHECK-EH-03: invoke.cont1: // CHECK-EH-03-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-03: return: @@ -2813,10 +2813,10 @@ X test22() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-03-NEXT: [[LPAD_VAL2:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1 // CHECK-EH-03-NEXT: resume { ptr, i32 } [[LPAD_VAL2]] // CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP5:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-03-NEXT: catch ptr null -// CHECK-EH-03-NEXT: [[TMP6:%.*]] = extractvalue { ptr, i32 } [[TMP5]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP6]]) #[[ATTR6]] +// CHECK-EH-03-NEXT: [[TMP4:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-03-NEXT: catch ptr null +// CHECK-EH-03-NEXT: [[TMP5:%.*]] = extractvalue { ptr, i32 } [[TMP4]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(ptr [[TMP5]]) #[[ATTR6]] // CHECK-EH-03-NEXT: unreachable // // CHECK-EH-11-LABEL: @_Z6test23b( @@ -2830,8 +2830,8 @@ X test22() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-11-NEXT: store ptr [[AGG_RESULT:%.*]], ptr [[RESULT_PTR]], align 4 // CHECK-EH-11-NEXT: [[FROMBOOL:%.*]] = zext i1 [[B:%.*]] to i8 // CHECK-EH-11-NEXT: store i8 [[FROMBOOL]], ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR]], align 1 -// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP1]] to i1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = load i8, ptr [[B_ADDR]], align 1 +// CHECK-EH-11-NEXT: [[TOBOOL:%.*]] = trunc i8 [[TMP0]] to i1 // CHECK-EH-11-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK-EH-11: if.then: // CHECK-EH-11-NEXT: store i1 false, ptr [[NRVO]], align 1 @@ -2847,17 +2847,17 @@ X test22() { // http://wg21.link/p2025r2#ex-19 // CHECK-EH-11: if.end: // CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) // CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERVKS_(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] // CHECK-EH-11: invoke.cont: // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[RETURN]] // CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP2:%.*]] = landingpad { ptr, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 0 -// CHECK-EH-11-NEXT: store ptr [[TMP3]], ptr [[EXN_SLOT]], align 4 -// CHECK-EH-11-NEXT: [[TMP4:%.*]] = extractvalue { ptr, i32 } [[TMP2]], 1 -// CHECK-EH-11-NEXT: store i32 [[TMP4]], ptr [[EHSELECTOR_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { ptr, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: [[TMP2:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 0 +// CHECK-EH-11-NEXT: store ptr [[TMP2]], ptr [[EXN_SLOT]], align 4 +// CHECK-EH-11-NEXT: [[TMP3:%.*]] = extractvalue { ptr, i32 } [[TMP1]], 1 +// CHECK-EH-11-NEXT: store i32 [[TMP3]], ptr [[EHSELECTOR_SLOT]], align 4 // CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[Y]]) #[[ATTR6]] // CHECK-EH-11-NEXT: br label [[EH_RESUME:%.*]] // CHECK-EH-11: return: diff --git a/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp b/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp index 9886b71dc135..7afaf035988f 100644 --- a/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp +++ b/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp @@ -351,7 +351,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZplRK1SS1_ -// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR7:[0-9]+]] { +// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR1]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 @@ -365,7 +365,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZN1SaSERKS_ -// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR7]] align 2 { +// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR1]] align 2 { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 @@ -433,7 +433,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@main.omp_outlined -// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR8:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR7:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8 @@ -503,7 +503,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@.omp_task_privates_map. -// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR9:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR8:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca ptr, align 8 @@ -565,28 +565,28 @@ int main(int argc, char **argv) { // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META8:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META10:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META12:![0-9]+]]) -// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14:![0-9]+]] +// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[TMP19]], i32 0, i32 1 // CHECK1-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8 -// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: call void [[TMP22]](ptr [[TMP23]], ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]]) #[[ATTR3]] -// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP26:%.*]] = load ptr, ptr [[TMP19]], align 8 // CHECK1-NEXT: [[TMP27:%.*]] = load ptr, ptr [[TMP24]], align 8 -// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[TMP29:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP27]], ptr [[TMP26]]) // CHECK1-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT_ANON]], ptr [[TMP19]], i32 0, i32 2 // CHECK1-NEXT: [[TMP31:%.*]] = load ptr, ptr [[TMP30]], align 8 @@ -596,19 +596,19 @@ int main(int argc, char **argv) { // CHECK1-NEXT: store i64 [[TMP33]], ptr [[TMP34]], align 8 // CHECK1-NEXT: [[TMP35:%.*]] = load ptr, ptr [[TMP25]], align 8 // CHECK1-NEXT: [[TMP36:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP35]], ptr [[TMP31]]) -// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[CONV_I:%.*]] = trunc i64 [[TMP37]] to i32 -// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I:%.*]] // CHECK1: omp.inner.for.cond.i: -// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[CONV2_I:%.*]] = sext i32 [[TMP38]] to i64 -// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[CMP_I:%.*]] = icmp ule i64 [[CONV2_I]], [[TMP39]] // CHECK1-NEXT: br i1 [[CMP_I]], label [[OMP_INNER_FOR_BODY_I:%.*]], label [[DOTOMP_OUTLINED__EXIT:%.*]] // CHECK1: omp.inner.for.body.i: -// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14 -// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] +// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP29]], align 4 // CHECK1-NEXT: [[IDXPROM_I:%.*]] = sext i32 [[TMP41]] to i64 // CHECK1-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds i16, ptr [[TMP36]], i64 [[IDXPROM_I]] @@ -617,9 +617,9 @@ int main(int argc, char **argv) { // CHECK1-NEXT: [[TMP43:%.*]] = load i32, ptr [[TMP29]], align 4 // CHECK1-NEXT: [[ADD4_I:%.*]] = add nsw i32 [[TMP43]], [[CONV3_I]] // CHECK1-NEXT: store i32 [[ADD4_I]], ptr [[TMP29]], align 4 -// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[ADD5_I:%.*]] = add nsw i32 [[TMP44]], 1 -// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I]] // CHECK1: .omp_outlined..exit: // CHECK1-NEXT: ret i32 0 diff --git a/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp b/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp index 76e5794b6fa5..5d6404cd3e54 100644 --- a/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp +++ b/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp @@ -351,7 +351,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZplRK1SS1_ -// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR7:[0-9]+]] { +// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR1]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 @@ -365,7 +365,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZN1SaSERKS_ -// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR7]] align 2 { +// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR1]] align 2 { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 @@ -433,7 +433,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@main.omp_outlined -// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR8:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR7:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8 @@ -503,7 +503,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@.omp_task_privates_map. -// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR9:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR8:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca ptr, align 8 @@ -565,28 +565,28 @@ int main(int argc, char **argv) { // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META8:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META10:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META12:![0-9]+]]) -// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14:![0-9]+]] +// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[TMP19]], i32 0, i32 1 // CHECK1-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8 -// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: call void [[TMP22]](ptr [[TMP23]], ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]]) #[[ATTR3]] -// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP26:%.*]] = load ptr, ptr [[TMP19]], align 8 // CHECK1-NEXT: [[TMP27:%.*]] = load ptr, ptr [[TMP24]], align 8 -// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[TMP29:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP27]], ptr [[TMP26]]) // CHECK1-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT_ANON]], ptr [[TMP19]], i32 0, i32 2 // CHECK1-NEXT: [[TMP31:%.*]] = load ptr, ptr [[TMP30]], align 8 @@ -596,19 +596,19 @@ int main(int argc, char **argv) { // CHECK1-NEXT: store i64 [[TMP33]], ptr [[TMP34]], align 8 // CHECK1-NEXT: [[TMP35:%.*]] = load ptr, ptr [[TMP25]], align 8 // CHECK1-NEXT: [[TMP36:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP35]], ptr [[TMP31]]) -// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[CONV_I:%.*]] = trunc i64 [[TMP37]] to i32 -// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I:%.*]] // CHECK1: omp.inner.for.cond.i: -// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15:![0-9]+]] +// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15:![0-9]+]] // CHECK1-NEXT: [[CONV2_I:%.*]] = sext i32 [[TMP38]] to i64 -// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[CMP_I:%.*]] = icmp ule i64 [[CONV2_I]], [[TMP39]] // CHECK1-NEXT: br i1 [[CMP_I]], label [[OMP_INNER_FOR_BODY_I:%.*]], label [[DOTOMP_OUTLINED__EXIT:%.*]] // CHECK1: omp.inner.for.body.i: -// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] -// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP29]], align 4, !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[IDXPROM_I:%.*]] = sext i32 [[TMP41]] to i64 // CHECK1-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds i16, ptr [[TMP36]], i64 [[IDXPROM_I]] @@ -617,9 +617,9 @@ int main(int argc, char **argv) { // CHECK1-NEXT: [[TMP43:%.*]] = load i32, ptr [[TMP29]], align 4, !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[ADD4_I:%.*]] = add nsw i32 [[TMP43]], [[CONV3_I]] // CHECK1-NEXT: store i32 [[ADD4_I]], ptr [[TMP29]], align 4, !llvm.access.group [[ACC_GRP15]] -// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[ADD5_I:%.*]] = add nsw i32 [[TMP44]], 1 -// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I]], !llvm.loop [[LOOP16:![0-9]+]] // CHECK1: .omp_outlined..exit: // CHECK1-NEXT: ret i32 0 diff --git a/clang/test/OpenMP/task_in_reduction_codegen.cpp b/clang/test/OpenMP/task_in_reduction_codegen.cpp index 41e6eaa1d688..82d203894364 100644 --- a/clang/test/OpenMP/task_in_reduction_codegen.cpp +++ b/clang/test/OpenMP/task_in_reduction_codegen.cpp @@ -373,7 +373,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZplRK1SS1_ -// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR7:[0-9]+]] { +// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR1]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 @@ -387,7 +387,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZN1SaSERKS_ -// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR7]] align 2 { +// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR1]] align 2 { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 @@ -455,7 +455,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@main.omp_outlined -// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR8:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR7:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8 @@ -506,7 +506,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@.omp_task_privates_map. -// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR9:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR8:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca ptr, align 8 @@ -550,23 +550,23 @@ int main(int argc, char **argv) { // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META6:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META8:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META10:![0-9]+]]) -// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !12 -// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias !12 -// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !12 -// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !12 -// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias !12 -// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !12 -// CHECK1-NEXT: [[TMP9:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !12 +// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META12:![0-9]+]] +// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias [[META12]] +// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META12]] +// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META12]] +// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias [[META12]] +// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META12]] +// CHECK1-NEXT: [[TMP9:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META12]] // CHECK1-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[TMP9]], i32 0, i32 1 // CHECK1-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 8 -// CHECK1-NEXT: [[TMP12:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !12 -// CHECK1-NEXT: [[TMP13:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !12 +// CHECK1-NEXT: [[TMP12:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META12]] +// CHECK1-NEXT: [[TMP13:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META12]] // CHECK1-NEXT: call void [[TMP12]](ptr [[TMP13]], ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]]) #[[ATTR3]] -// CHECK1-NEXT: [[TMP14:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias !12 -// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias !12 +// CHECK1-NEXT: [[TMP14:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias [[META12]] +// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias [[META12]] // CHECK1-NEXT: [[TMP16:%.*]] = load ptr, ptr [[TMP9]], align 8 // CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[TMP14]], align 8 -// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !12 +// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META12]] // CHECK1-NEXT: [[TMP19:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP18]], ptr [[TMP17]], ptr [[TMP16]]) // CHECK1-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANON]], ptr [[TMP9]], i32 0, i32 2 // CHECK1-NEXT: [[TMP21:%.*]] = load ptr, ptr [[TMP20]], align 8 @@ -610,15 +610,15 @@ int main(int argc, char **argv) { // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META18:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META20:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META22:![0-9]+]]) -// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !24 -// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias !24 -// CHECK1-NEXT: store ptr null, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !24 -// CHECK1-NEXT: store ptr null, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !24 -// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias !24 -// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !24 -// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !24 +// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META24:![0-9]+]] +// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias [[META24]] +// CHECK1-NEXT: store ptr null, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META24]] +// CHECK1-NEXT: store ptr null, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META24]] +// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias [[META24]] +// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META24]] +// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META24]] // CHECK1-NEXT: [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8 -// CHECK1-NEXT: [[TMP10:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !24 +// CHECK1-NEXT: [[TMP10:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META24]] // CHECK1-NEXT: [[TMP11:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP10]], ptr null, ptr [[TMP9]]) // CHECK1-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 // CHECK1-NEXT: [[INC_I:%.*]] = add nsw i32 [[TMP12]], 1 diff --git a/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp b/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp index 73ca0a7a4ca9..e48035a71eea 100644 --- a/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp +++ b/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp @@ -351,7 +351,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZplRK1SS1_ -// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR7:[0-9]+]] { +// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR1]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 @@ -365,7 +365,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZN1SaSERKS_ -// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR7]] align 2 { +// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR1]] align 2 { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 @@ -433,7 +433,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@main.omp_outlined -// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR8:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR7:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8 @@ -496,7 +496,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@.omp_task_privates_map. -// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR9:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR8:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca ptr, align 8 @@ -558,28 +558,28 @@ int main(int argc, char **argv) { // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META8:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META10:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META12:![0-9]+]]) -// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14:![0-9]+]] +// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[TMP19]], i32 0, i32 1 // CHECK1-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8 -// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: call void [[TMP22]](ptr [[TMP23]], ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]]) #[[ATTR3]] -// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP26:%.*]] = load ptr, ptr [[TMP19]], align 8 // CHECK1-NEXT: [[TMP27:%.*]] = load ptr, ptr [[TMP24]], align 8 -// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[TMP29:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP27]], ptr [[TMP26]]) // CHECK1-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT_ANON]], ptr [[TMP19]], i32 0, i32 2 // CHECK1-NEXT: [[TMP31:%.*]] = load ptr, ptr [[TMP30]], align 8 @@ -589,19 +589,19 @@ int main(int argc, char **argv) { // CHECK1-NEXT: store i64 [[TMP33]], ptr [[TMP34]], align 8 // CHECK1-NEXT: [[TMP35:%.*]] = load ptr, ptr [[TMP25]], align 8 // CHECK1-NEXT: [[TMP36:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP35]], ptr [[TMP31]]) -// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[CONV_I:%.*]] = trunc i64 [[TMP37]] to i32 -// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I:%.*]] // CHECK1: omp.inner.for.cond.i: -// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[CONV2_I:%.*]] = sext i32 [[TMP38]] to i64 -// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[CMP_I:%.*]] = icmp ule i64 [[CONV2_I]], [[TMP39]] // CHECK1-NEXT: br i1 [[CMP_I]], label [[OMP_INNER_FOR_BODY_I:%.*]], label [[DOTOMP_OUTLINED__EXIT:%.*]] // CHECK1: omp.inner.for.body.i: -// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14 -// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] +// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP29]], align 4 // CHECK1-NEXT: [[IDXPROM_I:%.*]] = sext i32 [[TMP41]] to i64 // CHECK1-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds i16, ptr [[TMP36]], i64 [[IDXPROM_I]] @@ -610,9 +610,9 @@ int main(int argc, char **argv) { // CHECK1-NEXT: [[TMP43:%.*]] = load i32, ptr [[TMP29]], align 4 // CHECK1-NEXT: [[ADD4_I:%.*]] = add nsw i32 [[TMP43]], [[CONV3_I]] // CHECK1-NEXT: store i32 [[ADD4_I]], ptr [[TMP29]], align 4 -// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[ADD5_I:%.*]] = add nsw i32 [[TMP44]], 1 -// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I]] // CHECK1: .omp_outlined..exit: // CHECK1-NEXT: ret i32 0 diff --git a/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp b/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp index 85df70f25a8b..e4d24fa48244 100644 --- a/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp +++ b/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp @@ -351,7 +351,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZplRK1SS1_ -// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR7:[0-9]+]] { +// CHECK1-SAME: (ptr noalias sret([[STRUCT_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], ptr nonnull align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR1]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 @@ -365,7 +365,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@_ZN1SaSERKS_ -// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR7]] align 2 { +// CHECK1-SAME: (ptr nonnull align 4 dereferenceable(4) [[THIS:%.*]], ptr nonnull align 4 dereferenceable(4) [[TMP0:%.*]]) #[[ATTR1]] align 2 { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 @@ -433,7 +433,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@main.omp_outlined -// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR8:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[DOTGLOBAL_TID_:%.*]], ptr noalias [[DOTBOUND_TID_:%.*]], ptr nonnull align 4 dereferenceable(4) [[A:%.*]], i64 [[VLA:%.*]], ptr nonnull align 2 dereferenceable(2) [[D:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_:%.*]], ptr nonnull align 8 dereferenceable(8) [[DOTTASK_RED_1:%.*]]) #[[ATTR7:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTGLOBAL_TID__ADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTBOUND_TID__ADDR:%.*]] = alloca ptr, align 8 @@ -496,7 +496,7 @@ int main(int argc, char **argv) { // // // CHECK1-LABEL: define {{[^@]+}}@.omp_task_privates_map. -// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR9:[0-9]+]] { +// CHECK1-SAME: (ptr noalias [[TMP0:%.*]], ptr noalias [[TMP1:%.*]], ptr noalias [[TMP2:%.*]]) #[[ATTR8:[0-9]+]] { // CHECK1-NEXT: entry: // CHECK1-NEXT: [[DOTADDR:%.*]] = alloca ptr, align 8 // CHECK1-NEXT: [[DOTADDR1:%.*]] = alloca ptr, align 8 @@ -558,28 +558,28 @@ int main(int argc, char **argv) { // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META8:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META10:![0-9]+]]) // CHECK1-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META12:![0-9]+]]) -// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: store i32 [[TMP2]], ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14:![0-9]+]] +// CHECK1-NEXT: store ptr [[TMP5]], ptr [[DOTPART_ID__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP8]], ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr @.omp_task_privates_map., ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP3]], ptr [[DOTTASK_T__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP10]], ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP12]], ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i64 [[TMP14]], ptr [[DOTST__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store i32 [[TMP16]], ptr [[DOTLITER__ADDR_I]], align 4, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP18]], ptr [[DOTREDUCTIONS__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: store ptr [[TMP7]], ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP19:%.*]] = load ptr, ptr [[__CONTEXT_ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[TMP19]], i32 0, i32 1 // CHECK1-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8 -// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP22:%.*]] = load ptr, ptr [[DOTCOPY_FN__ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP23:%.*]] = load ptr, ptr [[DOTPRIVATES__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: call void [[TMP22]](ptr [[TMP23]], ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]]) #[[ATTR3]] -// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias !14 -// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP24:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR_I]], align 8, !noalias [[META14]] +// CHECK1-NEXT: [[TMP25:%.*]] = load ptr, ptr [[DOTFIRSTPRIV_PTR_ADDR1_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[TMP26:%.*]] = load ptr, ptr [[TMP19]], align 8 // CHECK1-NEXT: [[TMP27:%.*]] = load ptr, ptr [[TMP24]], align 8 -// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias !14 +// CHECK1-NEXT: [[TMP28:%.*]] = load i32, ptr [[DOTGLOBAL_TID__ADDR_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: [[TMP29:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP27]], ptr [[TMP26]]) // CHECK1-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT_ANON]], ptr [[TMP19]], i32 0, i32 2 // CHECK1-NEXT: [[TMP31:%.*]] = load ptr, ptr [[TMP30]], align 8 @@ -589,19 +589,19 @@ int main(int argc, char **argv) { // CHECK1-NEXT: store i64 [[TMP33]], ptr [[TMP34]], align 8 // CHECK1-NEXT: [[TMP35:%.*]] = load ptr, ptr [[TMP25]], align 8 // CHECK1-NEXT: [[TMP36:%.*]] = call ptr @__kmpc_task_reduction_get_th_data(i32 [[TMP28]], ptr [[TMP35]], ptr [[TMP31]]) -// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias !14 +// CHECK1-NEXT: [[TMP37:%.*]] = load i64, ptr [[DOTLB__ADDR_I]], align 8, !noalias [[META14]] // CHECK1-NEXT: [[CONV_I:%.*]] = trunc i64 [[TMP37]] to i32 -// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14 +// CHECK1-NEXT: store i32 [[CONV_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I:%.*]] // CHECK1: omp.inner.for.cond.i: -// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15:![0-9]+]] +// CHECK1-NEXT: [[TMP38:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15:![0-9]+]] // CHECK1-NEXT: [[CONV2_I:%.*]] = sext i32 [[TMP38]] to i64 -// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: [[TMP39:%.*]] = load i64, ptr [[DOTUB__ADDR_I]], align 8, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[CMP_I:%.*]] = icmp ule i64 [[CONV2_I]], [[TMP39]] // CHECK1-NEXT: br i1 [[CMP_I]], label [[OMP_INNER_FOR_BODY_I:%.*]], label [[DOTOMP_OUTLINED__EXIT:%.*]] // CHECK1: omp.inner.for.body.i: -// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] -// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: [[TMP40:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: store i32 [[TMP40]], ptr [[I_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP29]], align 4, !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[IDXPROM_I:%.*]] = sext i32 [[TMP41]] to i64 // CHECK1-NEXT: [[ARRAYIDX_I:%.*]] = getelementptr inbounds i16, ptr [[TMP36]], i64 [[IDXPROM_I]] @@ -610,9 +610,9 @@ int main(int argc, char **argv) { // CHECK1-NEXT: [[TMP43:%.*]] = load i32, ptr [[TMP29]], align 4, !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[ADD4_I:%.*]] = add nsw i32 [[TMP43]], [[CONV3_I]] // CHECK1-NEXT: store i32 [[ADD4_I]], ptr [[TMP29]], align 4, !llvm.access.group [[ACC_GRP15]] -// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: [[ADD5_I:%.*]] = add nsw i32 [[TMP44]], 1 -// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias !14, !llvm.access.group [[ACC_GRP15]] +// CHECK1-NEXT: store i32 [[ADD5_I]], ptr [[DOTOMP_IV_I]], align 4, !noalias [[META14]], !llvm.access.group [[ACC_GRP15]] // CHECK1-NEXT: br label [[OMP_INNER_FOR_COND_I]], !llvm.loop [[LOOP16:![0-9]+]] // CHECK1: .omp_outlined..exit: // CHECK1-NEXT: ret i32 0 -- GitLab From e2972389111e30a93a21cf8c5f4d2284cbb60268 Mon Sep 17 00:00:00 2001 From: Nishant Mittal Date: Fri, 5 Jan 2024 21:21:16 +0530 Subject: [PATCH 128/728] [libc][math] fix nan* death tests failing in asan builds (#77110) --- libc/test/src/math/smoke/nan_test.cpp | 2 ++ libc/test/src/math/smoke/nanf_test.cpp | 2 ++ libc/test/src/math/smoke/nanl_test.cpp | 2 ++ 3 files changed, 6 insertions(+) diff --git a/libc/test/src/math/smoke/nan_test.cpp b/libc/test/src/math/smoke/nan_test.cpp index ae6da143ae41..81e1400f0bb6 100644 --- a/libc/test/src/math/smoke/nan_test.cpp +++ b/libc/test/src/math/smoke/nan_test.cpp @@ -42,6 +42,8 @@ TEST_F(LlvmLibcNanTest, RandomString) { run_test("123 ", 0x7ff8000000000000); } +#ifndef LIBC_HAVE_ADDRESS_SANITIZER TEST_F(LlvmLibcNanTest, InvalidInput) { EXPECT_DEATH([] { LIBC_NAMESPACE::nan(nullptr); }, WITH_SIGNAL(SIGSEGV)); } +#endif // LIBC_HAVE_ADDRESS_SANITIZER diff --git a/libc/test/src/math/smoke/nanf_test.cpp b/libc/test/src/math/smoke/nanf_test.cpp index a602aa9f8ceb..1d337ecf1dcd 100644 --- a/libc/test/src/math/smoke/nanf_test.cpp +++ b/libc/test/src/math/smoke/nanf_test.cpp @@ -41,6 +41,8 @@ TEST_F(LlvmLibcNanfTest, RandomString) { run_test("123 ", 0x7fc00000); } +#ifndef LIBC_HAVE_ADDRESS_SANITIZER TEST_F(LlvmLibcNanfTest, InvalidInput) { EXPECT_DEATH([] { LIBC_NAMESPACE::nanf(nullptr); }, WITH_SIGNAL(SIGSEGV)); } +#endif // LIBC_HAVE_ADDRESS_SANITIZER diff --git a/libc/test/src/math/smoke/nanl_test.cpp b/libc/test/src/math/smoke/nanl_test.cpp index 0a57b567916c..009710b49c83 100644 --- a/libc/test/src/math/smoke/nanl_test.cpp +++ b/libc/test/src/math/smoke/nanl_test.cpp @@ -67,6 +67,8 @@ TEST_F(LlvmLibcNanlTest, RandomString) { run_test("123 ", expected); } +#ifndef LIBC_HAVE_ADDRESS_SANITIZER TEST_F(LlvmLibcNanlTest, InvalidInput) { EXPECT_DEATH([] { LIBC_NAMESPACE::nanl(nullptr); }, WITH_SIGNAL(SIGSEGV)); } +#endif // LIBC_HAVE_ADDRESS_SANITIZER -- GitLab From 5352ce32fccd37c54aff2b3a2b0e3ca5115bb8a9 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 5 Jan 2024 08:19:04 -0800 Subject: [PATCH 129/728] [libc] fix -Warray-bounds in block_offset (#77001) GCC reports an instance of -Warray-bounds in block_offset. Reimplement block_offset in terms of memcpy_inline which was created to avoid this diagnostic. See the linked issue for the full trace of diagnostic. Fixes: https://github.com/llvm/llvm-project/issues/76877 --- libc/src/string/memory_utils/op_builtin.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/libc/src/string/memory_utils/op_builtin.h b/libc/src/string/memory_utils/op_builtin.h index 16c9f519c37e..3c17eef781e5 100644 --- a/libc/src/string/memory_utils/op_builtin.h +++ b/libc/src/string/memory_utils/op_builtin.h @@ -26,13 +26,7 @@ template struct Memcpy { static constexpr size_t SIZE = Size; LIBC_INLINE static void block_offset(Ptr __restrict dst, CPtr __restrict src, size_t offset) { -#ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE - return __builtin_memcpy_inline(dst + offset, src + offset, SIZE); -#else - // The codegen may be suboptimal. - for (size_t i = 0; i < Size; ++i) - dst[i + offset] = src[i + offset]; -#endif + memcpy_inline(dst + offset, src + offset); } LIBC_INLINE static void block(Ptr __restrict dst, CPtr __restrict src) { -- GitLab From 3b337bbc811002d088d3ae6fcae869a8d5682bc1 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 5 Jan 2024 10:24:30 -0600 Subject: [PATCH 130/728] [ELF] Attempt to set the OS when using 'makeTriple()' (#76992) Summary: This patch fixes up the `makeTriple()` interface to emit append the operating system information when it is readily avaialble from the ELF. The main motivation for this is so the GPU architectures can be easily identified correctly when given and ELF. E.g. we want `amdgpu-amd-amdhsa` as the output and not `amdgpu--`. This required adding support for the CUDA OS/ABI, which is easily found to be `0x33` when using `readelf`. --- llvm/include/llvm/BinaryFormat/ELF.h | 1 + llvm/include/llvm/Object/ELFObjectFile.h | 30 +++++++++++++++ llvm/include/llvm/Object/ObjectFile.h | 1 + llvm/lib/Object/ObjectFile.cpp | 11 +++++- llvm/tools/llvm-readobj/ELFDumper.cpp | 5 +++ llvm/unittests/Object/ELFObjectFileTest.cpp | 41 +++++++++++++++++---- 6 files changed, 80 insertions(+), 9 deletions(-) diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 0f968eac36e7..9b8128a9ec40 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -356,6 +356,7 @@ enum { ELFOSABI_AROS = 15, // AROS ELFOSABI_FENIXOS = 16, // FenixOS ELFOSABI_CLOUDABI = 17, // Nuxi CloudABI + ELFOSABI_CUDA = 51, // NVIDIA CUDA architecture. ELFOSABI_FIRST_ARCH = 64, // First architecture-specific OS ABI ELFOSABI_AMDGPU_HSA = 64, // AMD HSA runtime ELFOSABI_AMDGPU_PAL = 65, // AMD PAL runtime diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index da78e11b678d..da72b0e335b9 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -454,6 +454,7 @@ public: uint8_t getBytesInAddress() const override; StringRef getFileFormatName() const override; Triple::ArchType getArch() const override; + Triple::OSType getOS() const override; Expected getStartAddress() const override; unsigned getPlatformFlags() const override { return EF.getHeader().e_flags; } @@ -1382,6 +1383,35 @@ template Triple::ArchType ELFObjectFile::getArch() const { } } +template Triple::OSType ELFObjectFile::getOS() const { + switch (EF.getHeader().e_ident[ELF::EI_OSABI]) { + case ELF::ELFOSABI_NETBSD: + return Triple::NetBSD; + case ELF::ELFOSABI_LINUX: + return Triple::Linux; + case ELF::ELFOSABI_HURD: + return Triple::Hurd; + case ELF::ELFOSABI_SOLARIS: + return Triple::Solaris; + case ELF::ELFOSABI_AIX: + return Triple::AIX; + case ELF::ELFOSABI_FREEBSD: + return Triple::FreeBSD; + case ELF::ELFOSABI_OPENBSD: + return Triple::OpenBSD; + case ELF::ELFOSABI_CUDA: + return Triple::CUDA; + case ELF::ELFOSABI_AMDGPU_HSA: + return Triple::AMDHSA; + case ELF::ELFOSABI_AMDGPU_PAL: + return Triple::AMDPAL; + case ELF::ELFOSABI_AMDGPU_MESA3D: + return Triple::Mesa3D; + default: + return Triple::UnknownOS; + } +} + template Expected ELFObjectFile::getStartAddress() const { return EF.getHeader().e_entry; diff --git a/llvm/include/llvm/Object/ObjectFile.h b/llvm/include/llvm/Object/ObjectFile.h index c254fc2ccfde..8c868c7643ed 100644 --- a/llvm/include/llvm/Object/ObjectFile.h +++ b/llvm/include/llvm/Object/ObjectFile.h @@ -337,6 +337,7 @@ public: virtual StringRef getFileFormatName() const = 0; virtual Triple::ArchType getArch() const = 0; + virtual Triple::OSType getOS() const { return Triple::UnknownOS; } virtual Expected getFeatures() const = 0; virtual std::optional tryGetCPUName() const { return std::nullopt; diff --git a/llvm/lib/Object/ObjectFile.cpp b/llvm/lib/Object/ObjectFile.cpp index ca921836b7f6..c05eb0a0468e 100644 --- a/llvm/lib/Object/ObjectFile.cpp +++ b/llvm/lib/Object/ObjectFile.cpp @@ -111,6 +111,10 @@ Triple ObjectFile::makeTriple() const { auto Arch = getArch(); TheTriple.setArch(Triple::ArchType(Arch)); + auto OS = getOS(); + if (OS != Triple::UnknownOS) + TheTriple.setOS(OS); + // For ARM targets, try to use the build attributes to build determine // the build target. Target features are also added, but later during // disassembly. @@ -129,10 +133,13 @@ Triple ObjectFile::makeTriple() const { // XCOFF implies AIX. TheTriple.setOS(Triple::AIX); TheTriple.setObjectFormat(Triple::XCOFF); - } - else if (isGOFF()) { + } else if (isGOFF()) { TheTriple.setOS(Triple::ZOS); TheTriple.setObjectFormat(Triple::GOFF); + } else if (TheTriple.isAMDGPU()) { + TheTriple.setVendor(Triple::AMD); + } else if (TheTriple.isNVPTX()) { + TheTriple.setVendor(Triple::NVIDIA); } return TheTriple; diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index abf7ba6ba1c3..10797b83d3d9 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -1084,6 +1084,7 @@ const EnumEntry ElfOSABI[] = { {"AROS", "AROS", ELF::ELFOSABI_AROS}, {"FenixOS", "FenixOS", ELF::ELFOSABI_FENIXOS}, {"CloudABI", "CloudABI", ELF::ELFOSABI_CLOUDABI}, + {"CUDA", "NVIDIA - CUDA", ELF::ELFOSABI_CUDA}, {"Standalone", "Standalone App", ELF::ELFOSABI_STANDALONE} }; @@ -1093,6 +1094,10 @@ const EnumEntry AMDGPUElfOSABI[] = { {"AMDGPU_MESA3D", "AMDGPU - MESA3D", ELF::ELFOSABI_AMDGPU_MESA3D} }; +const EnumEntry NVPTXElfOSABI[] = { + {"NVIDIA_CUDA", "NVIDIA - CUDA", ELF::ELFOSABI_CUDA}, +}; + const EnumEntry ARMElfOSABI[] = { {"ARM", "ARM", ELF::ELFOSABI_ARM} }; diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp index 2878ca088cd7..e7619ded494e 100644 --- a/llvm/unittests/Object/ELFObjectFileTest.cpp +++ b/llvm/unittests/Object/ELFObjectFileTest.cpp @@ -36,7 +36,8 @@ struct DataForTest { template std::vector makeElfData(uint8_t Class, uint8_t Encoding, - uint16_t Machine) { + uint16_t Machine, uint8_t OS, + uint16_t Flags) { T Ehdr{}; // Zero-initialise the header. Ehdr.e_ident[ELF::EI_MAG0] = 0x7f; Ehdr.e_ident[ELF::EI_MAG1] = 'E'; @@ -45,9 +46,11 @@ struct DataForTest { Ehdr.e_ident[ELF::EI_CLASS] = Class; Ehdr.e_ident[ELF::EI_DATA] = Encoding; Ehdr.e_ident[ELF::EI_VERSION] = 1; + Ehdr.e_ident[ELF::EI_OSABI] = OS; Ehdr.e_type = ELF::ET_REL; Ehdr.e_machine = Machine; Ehdr.e_version = 1; + Ehdr.e_flags = Flags; Ehdr.e_ehsize = sizeof(T); bool IsLittleEndian = Encoding == ELF::ELFDATA2LSB; @@ -64,12 +67,13 @@ struct DataForTest { return Bytes; } - DataForTest(uint8_t Class, uint8_t Encoding, uint16_t Machine) { + DataForTest(uint8_t Class, uint8_t Encoding, uint16_t Machine, + uint8_t OS = ELF::ELFOSABI_NONE, uint16_t Flags = 0) { if (Class == ELF::ELFCLASS64) - Data = makeElfData(Class, Encoding, Machine); + Data = makeElfData(Class, Encoding, Machine, OS, Flags); else { assert(Class == ELF::ELFCLASS32); - Data = makeElfData(Class, Encoding, Machine); + Data = makeElfData(Class, Encoding, Machine, OS, Flags); } } }; @@ -287,6 +291,30 @@ TEST(ELFObjectFileTest, MachineTestForXtensa) { checkFormatAndArch(Data, Formats[Idx], Triple::xtensa); } +TEST(ELFObjectFileTest, CheckOSAndTriple) { + std::tuple Formats[] = { + {ELF::EM_AMDGPU, ELF::ELFOSABI_AMDGPU_HSA, "amdgcn-amd-amdhsa"}, + {ELF::EM_X86_64, ELF::ELFOSABI_LINUX, "x86_64--linux"}, + {ELF::EM_X86_64, ELF::ELFOSABI_NETBSD, "x86_64--netbsd"}, + {ELF::EM_X86_64, ELF::ELFOSABI_HURD, "x86_64--hurd"}, + {ELF::EM_X86_64, ELF::ELFOSABI_SOLARIS, "x86_64--solaris"}, + {ELF::EM_X86_64, ELF::ELFOSABI_AIX, "x86_64--aix"}, + {ELF::EM_X86_64, ELF::ELFOSABI_FREEBSD, "x86_64--freebsd"}, + {ELF::EM_X86_64, ELF::ELFOSABI_OPENBSD, "x86_64--openbsd"}, + {ELF::EM_CUDA, ELF::ELFOSABI_CUDA, "nvptx64-nvidia-cuda"}}; + for (auto [Machine, OS, Triple] : Formats) { + const DataForTest D(ELF::ELFCLASS64, ELF::ELFDATA2LSB, Machine, OS, + ELF::EF_AMDGPU_MACH_AMDGCN_LAST); + Expected> ELFObjOrErr = + object::ObjectFile::createELFObjectFile( + MemoryBufferRef(toStringRef(D.Data), "dummyELF")); + ASSERT_THAT_EXPECTED(ELFObjOrErr, Succeeded()); + + auto &ELFObj = **ELFObjOrErr; + EXPECT_EQ(Triple, ELFObj.makeTriple().getTriple()); + } +} + // ELF relative relocation type test. TEST(ELFObjectFileTest, RelativeRelocationTypeTest) { EXPECT_EQ(ELF::R_CKCORE_RELATIVE, getELFRelativeRelocationType(ELF::EM_CSKY)); @@ -1273,13 +1301,12 @@ Sections: )"; auto ErroringMatcher = [](const Elf_Shdr &Sec) -> Expected { - if(Sec.sh_type == ELF::SHT_PROGBITS) + if (Sec.sh_type == ELF::SHT_PROGBITS) return createError("This was supposed to fail."); return false; }; - DoCheckFails(OneTextSection, ErroringMatcher, - "This was supposed to fail."); + DoCheckFails(OneTextSection, ErroringMatcher, "This was supposed to fail."); StringRef MissingRelocatableContent = R"( Sections: -- GitLab From 59567e711d262813b6708f1a242433a401c03e9f Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 5 Jan 2024 10:58:10 -0600 Subject: [PATCH 131/728] [ELF][Obvious] Remove unused CUDA OS/ABI struct Summary: This did not need a separate struct because it is not past the "architecture specific" ones in the identifier. This was accidentally left when it was added somewhere else and caused unused variable warnings. --- llvm/tools/llvm-readobj/ELFDumper.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index 10797b83d3d9..f369a63add11 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -1094,10 +1094,6 @@ const EnumEntry AMDGPUElfOSABI[] = { {"AMDGPU_MESA3D", "AMDGPU - MESA3D", ELF::ELFOSABI_AMDGPU_MESA3D} }; -const EnumEntry NVPTXElfOSABI[] = { - {"NVIDIA_CUDA", "NVIDIA - CUDA", ELF::ELFOSABI_CUDA}, -}; - const EnumEntry ARMElfOSABI[] = { {"ARM", "ARM", ELF::ELFOSABI_ARM} }; -- GitLab From c49965b97e159b7ff92a5fca3e144b2d2574a84d Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Fri, 5 Jan 2024 09:10:03 -0800 Subject: [PATCH 132/728] [mlgo] Fix post PR #76919 Relaxed a bit the opcode checks to make the test less sensitive to changes resulting in opcode numbering. --- llvm/test/CodeGen/MLRegAlloc/dev-mode-extra-features-logging.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/CodeGen/MLRegAlloc/dev-mode-extra-features-logging.ll b/llvm/test/CodeGen/MLRegAlloc/dev-mode-extra-features-logging.ll index 98cf234d83a2..79be2447abcf 100644 --- a/llvm/test/CodeGen/MLRegAlloc/dev-mode-extra-features-logging.ll +++ b/llvm/test/CodeGen/MLRegAlloc/dev-mode-extra-features-logging.ll @@ -26,7 +26,7 @@ ; Also, the first eviction problem is significantly less than 300 instructions. Check ; that there is a zero value. ; Note: we're regex-ing some of the opcodes to avoid test flakyness. -; CHECK: instructions: 19,{{([0-9]{4})}},17{{([0-9]{2})}},17{{([0-9]{2})}},{{.*}},0, +; CHECK: instructions: 19,{{([0-9]{4})}},1{{([0-9]{3})}},1{{([0-9]{3})}},{{.*}},0, ; Only the candidate virtreg and the 10th LR are included in this problem. Make ; sure the other LRs have values of zero. There are 2700 0s followed by some 1s. ; There's a limit to how many repetitions can be matched. -- GitLab From 4dd5d967975fa8d52b8c60596d892d9dd5615809 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 5 Jan 2024 09:22:54 -0800 Subject: [PATCH 133/728] [RISCV] Don't call use_nodbg_operands for physical registers in RISCVOptWInstrs hasAllNBitUsers. (#77032) The ADDIW in the new test case was incorrectly removed due to incorrectly following the x10 register from the return value back to the argument. This is due to use_nodbg_operands returning every instruction that uses a physical register regardless of the data flow. --- llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp | 6 +++++- llvm/test/CodeGen/RISCV/opt-w-instrs.mir | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index 2c2b34bb5b77..c16eee67f3c5 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -126,7 +126,11 @@ static bool hasAllNBitUsers(const MachineInstr &OrigMI, if (MI->getNumExplicitDefs() != 1) return false; - for (auto &UserOp : MRI.use_nodbg_operands(MI->getOperand(0).getReg())) { + Register DestReg = MI->getOperand(0).getReg(); + if (!DestReg.isVirtual()) + return false; + + for (auto &UserOp : MRI.use_nodbg_operands(DestReg)) { const MachineInstr *UserMI = UserOp.getParent(); unsigned OpIdx = UserOp.getOperandNo(); diff --git a/llvm/test/CodeGen/RISCV/opt-w-instrs.mir b/llvm/test/CodeGen/RISCV/opt-w-instrs.mir index 0ecf8fd6bef3..ebac5a42fbcd 100644 --- a/llvm/test/CodeGen/RISCV/opt-w-instrs.mir +++ b/llvm/test/CodeGen/RISCV/opt-w-instrs.mir @@ -28,3 +28,23 @@ body: | $x11 = COPY %4 PseudoRET ... + +--- +name: physreg +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10, $x11 + + ; CHECK-ZFA-LABEL: name: physreg + ; CHECK-ZFA: liveins: $x10, $x11 + ; CHECK-ZFA-NEXT: {{ $}} + ; CHECK-ZFA-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10 + ; CHECK-ZFA-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[COPY]], 0 + ; CHECK-ZFA-NEXT: $x10 = COPY [[ADDIW]] + ; CHECK-ZFA-NEXT: PseudoRET + %0:gpr = COPY $x10 + %1:gpr = ADDIW %0, 0 + $x10 = COPY %1 + PseudoRET +... -- GitLab From dfe9bb4dc22796073b677984f2ffa5580003ce82 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 5 Jan 2024 11:23:10 -0600 Subject: [PATCH 134/728] [ELF] Attempt to fix test on big endian architectures Summary: This test fails because AMDGPU has a check for little-endianness before returning the architecture. This test attempts to force the type to be considered little-endian for the purpose of this test. --- llvm/unittests/Object/ELFObjectFileTest.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp index e7619ded494e..ccf52ae48e92 100644 --- a/llvm/unittests/Object/ELFObjectFileTest.cpp +++ b/llvm/unittests/Object/ELFObjectFileTest.cpp @@ -305,12 +305,11 @@ TEST(ELFObjectFileTest, CheckOSAndTriple) { for (auto [Machine, OS, Triple] : Formats) { const DataForTest D(ELF::ELFCLASS64, ELF::ELFDATA2LSB, Machine, OS, ELF::EF_AMDGPU_MACH_AMDGCN_LAST); - Expected> ELFObjOrErr = - object::ObjectFile::createELFObjectFile( - MemoryBufferRef(toStringRef(D.Data), "dummyELF")); + Expected ELFObjOrErr = ELF64LEObjectFile::create( + MemoryBufferRef(toStringRef(D.Data), "dummyELF")); ASSERT_THAT_EXPECTED(ELFObjOrErr, Succeeded()); - auto &ELFObj = **ELFObjOrErr; + auto &ELFObj = *ELFObjOrErr; EXPECT_EQ(Triple, ELFObj.makeTriple().getTriple()); } } -- GitLab From e42edb5547618c172abe25914000bb61f5278c4c Mon Sep 17 00:00:00 2001 From: Greg Clayton Date: Fri, 5 Jan 2024 09:26:54 -0800 Subject: [PATCH 135/728] [lldb] Fix expressions that involve nested structs/classes/unions. (#77029) The LLDB expression parser relies on using the external AST source support in LLDB. This allows us to find a class at the root namespace level, but it wouldn't allow us to find nested classes all of the time. When LLDB finds a class via this mechanism, it would be able to complete this class when needed, but during completion, we wouldn't populate nested types within this class which would prevent us from finding contained types when needed as clang would expect them to be present if a class was completed. When we parse a type for a class, struct or union, we make a forward declaration to the class which can be completed. Now when the class is completed, we also add any contained types to the class' declaration context which now allows these types to be found. If we have a struct that contains a struct, we will add the forward declaration of the contained structure which can be c ompleted later. Having this forward declaration makes it possible for LLDB to find everything it needs now. This should fix an existing issue: https://github.com/llvm/llvm-project/issues/53904 Previously, contained types could be parsed by accident and allow expression to complete successfully. Other times we would have to run an expression multiple times because our old type lookup from our expressions would cau se a type to be parsed, but not used in the current expression, but this would have parsed a type into the containing decl context and the expression might succeed if it is run again. --- .../SymbolFile/DWARF/DWARFASTParserClang.cpp | 13 +++- .../SymbolFile/DWARF/DWARFASTParserClang.h | 1 + .../API/commands/expression/nested/Makefile | 3 + .../nested/TestNestedExpressions.py | 70 +++++++++++++++++++ .../API/commands/expression/nested/main.cpp | 31 ++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 lldb/test/API/commands/expression/nested/Makefile create mode 100644 lldb/test/API/commands/expression/nested/TestNestedExpressions.py create mode 100644 lldb/test/API/commands/expression/nested/main.cpp diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp index 3e08f2550081..009722b85aa1 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp @@ -2150,6 +2150,7 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die, SymbolFileDWARF *dwarf = die.GetDWARF(); ClangASTImporter::LayoutInfo layout_info; + std::vector contained_type_dies; if (die.HasChildren()) { const bool type_is_objc_object_or_interface = @@ -2175,7 +2176,8 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die, DelayedPropertyList delayed_properties; ParseChildMembers(die, clang_type, bases, member_function_dies, - delayed_properties, default_accessibility, layout_info); + contained_type_dies, delayed_properties, + default_accessibility, layout_info); // Now parse any methods if there were any... for (const DWARFDIE &die : member_function_dies) @@ -2231,6 +2233,12 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die, if (record_decl) GetClangASTImporter().SetRecordLayout(record_decl, layout_info); } + // Now parse all contained types inside of the class. We make forward + // declarations to all classes, but we need the CXXRecordDecl to have decls + // for all contained types because we don't get asked for them via the + // external AST support. + for (const DWARFDIE &die : contained_type_dies) + dwarf->ResolveType(die); return (bool)clang_type; } @@ -3110,6 +3118,7 @@ bool DWARFASTParserClang::ParseChildMembers( const DWARFDIE &parent_die, CompilerType &class_clang_type, std::vector> &base_classes, std::vector &member_function_dies, + std::vector &contained_type_dies, DelayedPropertyList &delayed_properties, const AccessType default_accessibility, ClangASTImporter::LayoutInfo &layout_info) { @@ -3159,6 +3168,8 @@ bool DWARFASTParserClang::ParseChildMembers( break; default: + if (llvm::dwarf::isType(tag)) + contained_type_dies.push_back(die); break; } } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h index 3e28e54d6220..8d4af203bb28 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h @@ -175,6 +175,7 @@ protected: lldb_private::CompilerType &class_compiler_type, std::vector> &base_classes, std::vector &member_function_dies, + std::vector &contained_type_dies, DelayedPropertyList &delayed_properties, const lldb::AccessType default_accessibility, lldb_private::ClangASTImporter::LayoutInfo &layout_info); diff --git a/lldb/test/API/commands/expression/nested/Makefile b/lldb/test/API/commands/expression/nested/Makefile new file mode 100644 index 000000000000..99998b20bcb0 --- /dev/null +++ b/lldb/test/API/commands/expression/nested/Makefile @@ -0,0 +1,3 @@ +CXX_SOURCES := main.cpp + +include Makefile.rules diff --git a/lldb/test/API/commands/expression/nested/TestNestedExpressions.py b/lldb/test/API/commands/expression/nested/TestNestedExpressions.py new file mode 100644 index 000000000000..7f194e921e56 --- /dev/null +++ b/lldb/test/API/commands/expression/nested/TestNestedExpressions.py @@ -0,0 +1,70 @@ +""" +Test calling an expression with errors that a FixIt can fix. +""" + +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class NestedExpressions(TestBase): + + def test_enum_in_nested_structs(self): + """ + Test expressions that references an enumeration in nested structs. + """ + self.build() + exe_path = self.getBuildArtifact("a.out") + target = self.dbg.CreateTarget(exe_path) + self.assertTrue(target, "Target: %s is not valid." % (exe_path)) + self.expect_expr("A::B::C::EnumType::Eleven", + result_type="A::B::C::EnumType", + result_value="Eleven") + + def test_struct_in_nested_structs(self): + """ + Test expressions that references a struct in nested structs. + """ + self.build() + exe_path = self.getBuildArtifact("a.out") + target = self.dbg.CreateTarget(exe_path) + self.assertTrue(target, "Target: %s is not valid." % (exe_path)) + self.expect_expr("sizeof(A::B::C)", result_value="1") + self.expect_expr("sizeof(A::B)", result_value="2") + + def test_static_in_nested_structs(self): + """ + Test expressions that references a static variable in nested structs. + """ + self.build() + (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint( + self, "Stop here to evaluate expressions", lldb.SBFileSpec("main.cpp") + ) + self.expect_expr("A::B::C::enum_static", + result_type="A::B::C::EnumType", + result_value="Eleven") + + def test_enum_in_nested_namespaces(self): + """ + Test expressions that references an enumeration in nested namespaces. + """ + self.build() + exe_path = self.getBuildArtifact("a.out") + target = self.dbg.CreateTarget(exe_path) + self.assertTrue(target, "Target: %s is not valid." % (exe_path)) + self.expect_expr("a::b::c::Color::Blue", + result_type="a::b::c::Color", + result_value="Blue") + + def test_static_in_nested_namespaces(self): + """ + Test expressions that references an enumeration in nested namespaces. + """ + self.build() + (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint( + self, "Stop here to evaluate expressions", lldb.SBFileSpec("main.cpp") + ) + self.expect_expr("a::b::c::d", + result_type="int", + result_value="12") diff --git a/lldb/test/API/commands/expression/nested/main.cpp b/lldb/test/API/commands/expression/nested/main.cpp new file mode 100644 index 000000000000..620535fa0953 --- /dev/null +++ b/lldb/test/API/commands/expression/nested/main.cpp @@ -0,0 +1,31 @@ +namespace a { +namespace b { +namespace c { +static int d = 12; +enum Color { Red, Green, Blue }; +} // namespace c +} // namespace b +} // namespace a + +struct A { + int _a = 'a'; + struct B { + short _b = 'b'; + struct C { + char _c = 'c'; + enum EnumType : int { Eleven = 11 }; + static EnumType enum_static; + }; + }; +}; + +A::B::C::EnumType A::B::C::enum_static = A::B::C::Eleven; + +int foo() { + a::b::c::Color color = a::b::c::Blue; + return A::B::C::enum_static == a::b::c::d && ((int)color == 0); +} + +int main() { + return foo(); // Stop here to evaluate expressions +} -- GitLab From 1259c0512292a6559fb073b16056bff2e934822b Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 6 Jan 2024 01:43:21 +0800 Subject: [PATCH 136/728] [InstCombine] Canonicalize `switch(X << C)` into `switch(X)` (#77068) This patch canonicalizes `switch(X << C)` to `switch(X)`. If the shift may wrap, an and instruction will be created to mask out all of the shifted bits. Alive2: https://alive2.llvm.org/ce/z/wSsL5y NOTE: We can relax the one-use constraint. But I don't see any benefit in my benchmark. Compile-time impact: http://llvm-compile-time-tracker.com/compare.php?from=a776740d6296520b8bde156aa3f8d9ecb32cddd9&to=6dd783b9f90ae5f258102d732953567d7e317c02&stat=instructions%3Au |stage1-O3|stage1-ReleaseThinLTO|stage1-ReleaseLTO-g|stage1-O0-g|stage2-O3|stage2-O0-g|stage2-clang| |--|--|--|--|--|--|--| |-0.00%|+0.01%|-0.02%|-0.01%|+0.02%|-0.00%|+0.01%| --- .../InstCombine/InstructionCombining.cpp | 27 +++ .../test/Transforms/InstCombine/switch-shl.ll | 185 ++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 llvm/test/Transforms/InstCombine/switch-shl.ll diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index b45c6784b85b..e845bf38df2d 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -3220,6 +3220,33 @@ Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) { return replaceOperand(SI, 0, Op0); } + uint64_t ShiftAmt; + if (match(Cond, m_Shl(m_Value(Op0), m_ConstantInt(ShiftAmt))) && + ShiftAmt < Op0->getType()->getScalarSizeInBits() && + all_of(SI.cases(), [&](const auto &Case) { + return Case.getCaseValue()->getValue().countr_zero() >= ShiftAmt; + })) { + // Change 'switch (X << 2) case 4:' into 'switch (X) case 1:'. + OverflowingBinaryOperator *Shl = cast(Cond); + if (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap() || + Shl->hasOneUse()) { + Value *NewCond = Op0; + if (!Shl->hasNoUnsignedWrap() && !Shl->hasNoSignedWrap()) { + // If the shift may wrap, we need to mask off the shifted bits. + unsigned BitWidth = Op0->getType()->getScalarSizeInBits(); + NewCond = Builder.CreateAnd( + Op0, APInt::getLowBitsSet(BitWidth, BitWidth - ShiftAmt)); + } + for (auto Case : SI.cases()) { + const APInt &CaseVal = Case.getCaseValue()->getValue(); + APInt ShiftedCase = Shl->hasNoSignedWrap() ? CaseVal.ashr(ShiftAmt) + : CaseVal.lshr(ShiftAmt); + Case.setValue(ConstantInt::get(SI.getContext(), ShiftedCase)); + } + return replaceOperand(SI, 0, NewCond); + } + } + KnownBits Known = computeKnownBits(Cond, 0, &SI); unsigned LeadingKnownZeros = Known.countMinLeadingZeros(); unsigned LeadingKnownOnes = Known.countMinLeadingOnes(); diff --git a/llvm/test/Transforms/InstCombine/switch-shl.ll b/llvm/test/Transforms/InstCombine/switch-shl.ll new file mode 100644 index 000000000000..037ef37f97a3 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/switch-shl.ll @@ -0,0 +1,185 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=instcombine -S | FileCheck %s + +define i1 @test_switch_with_shl_mask(i32 %a) { +; CHECK-LABEL: define i1 @test_switch_with_shl_mask( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 [[A]] to i8 +; CHECK-NEXT: switch i8 [[TRUNC]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i8 0, label [[SW_BB:%.*]] +; CHECK-NEXT: i8 1, label [[SW_BB]] +; CHECK-NEXT: i8 -128, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %b = shl i32 %a, 24 + switch i32 %b, label %sw.default [ + i32 0, label %sw.bb + i32 16777216, label %sw.bb + i32 2147483648, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +define i1 @test_switch_with_shl_nuw_multiuse(i32 %a) { +; CHECK-LABEL: define i1 @test_switch_with_shl_nuw_multiuse( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[B:%.*]] = shl nuw i32 [[A]], 24 +; CHECK-NEXT: call void @use(i32 [[B]]) +; CHECK-NEXT: switch i32 [[A]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 1, label [[SW_BB]] +; CHECK-NEXT: i32 128, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %b = shl nuw i32 %a, 24 + call void @use(i32 %b) + switch i32 %b, label %sw.default [ + i32 0, label %sw.bb + i32 16777216, label %sw.bb + i32 2147483648, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +define i1 @test_switch_with_shl_nsw_multiuse(i32 %a) { +; CHECK-LABEL: define i1 @test_switch_with_shl_nsw_multiuse( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[B:%.*]] = shl nsw i32 [[A]], 24 +; CHECK-NEXT: call void @use(i32 [[B]]) +; CHECK-NEXT: switch i32 [[A]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 1, label [[SW_BB]] +; CHECK-NEXT: i32 -128, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %b = shl nsw i32 %a, 24 + call void @use(i32 %b) + switch i32 %b, label %sw.default [ + i32 0, label %sw.bb + i32 16777216, label %sw.bb + i32 2147483648, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +; Negative tests + +define i1 @test_switch_with_shl_mask_multiuse(i32 %a) { +; CHECK-LABEL: define i1 @test_switch_with_shl_mask_multiuse( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[B:%.*]] = shl i32 [[A]], 24 +; CHECK-NEXT: call void @use(i32 [[B]]) +; CHECK-NEXT: switch i32 [[B]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 16777216, label [[SW_BB]] +; CHECK-NEXT: i32 -2147483648, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %b = shl i32 %a, 24 + call void @use(i32 %b) + switch i32 %b, label %sw.default [ + i32 0, label %sw.bb + i32 16777216, label %sw.bb + i32 2147483648, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +define i1 @test_switch_with_shl_mask_unknown_shamt(i32 %a, i32 %shamt) { +; CHECK-LABEL: define i1 @test_switch_with_shl_mask_unknown_shamt( +; CHECK-SAME: i32 [[A:%.*]], i32 [[SHAMT:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[B:%.*]] = shl i32 [[A]], [[SHAMT]] +; CHECK-NEXT: switch i32 [[B]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 16777216, label [[SW_BB]] +; CHECK-NEXT: i32 -2147483648, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %b = shl i32 %a, %shamt + switch i32 %b, label %sw.default [ + i32 0, label %sw.bb + i32 16777216, label %sw.bb + i32 2147483648, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +define i1 @test_switch_with_shl_mask_poison(i32 %a) { +; CHECK-LABEL: define i1 @test_switch_with_shl_mask_poison( +; CHECK-SAME: i32 [[A:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: switch i32 poison, label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 0, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 16777216, label [[SW_BB]] +; CHECK-NEXT: i32 -2147483648, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 true +; CHECK: sw.default: +; CHECK-NEXT: ret i1 false +; +entry: + %b = shl i32 %a, 32 + switch i32 %b, label %sw.default [ + i32 0, label %sw.bb + i32 16777216, label %sw.bb + i32 2147483648, label %sw.bb + ] + +sw.bb: + ret i1 true +sw.default: + ret i1 false +} + +declare void @use(i32) -- GitLab From 0e8b09c43d9f9e68c95fa1e6f6fb5c8307f3927d Mon Sep 17 00:00:00 2001 From: Erich Keane Date: Fri, 5 Jan 2024 09:43:39 -0800 Subject: [PATCH 137/728] [OpenACC] Implement 'default' clause parsing. (#77002) A simple clause that is permitted on a few different constructs, 'default' takes a required parameter of either 'none' or 'present'. This patch implements parsing for it. --- .../clang/Basic/DiagnosticParseKinds.td | 2 + clang/include/clang/Basic/OpenACCKinds.h | 12 +++ clang/lib/Parse/ParseOpenACC.cpp | 84 +++++++++++++++- clang/test/ParserOpenACC/parse-clauses.c | 96 +++++++++++++++++++ 4 files changed, 189 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index e4b1069cde18..088f8b74983c 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1364,6 +1364,8 @@ def err_acc_invalid_clause : Error<"invalid OpenACC clause %0">; def err_acc_missing_directive : Error<"expected OpenACC directive">; def err_acc_invalid_open_paren : Error<"expected clause-list or newline in OpenACC directive">; +def err_acc_invalid_default_clause_kind + : Error<"invalid value for 'default' clause; expected 'present' or 'none'">; // OpenMP support. def warn_pragma_omp_ignored : Warning< diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h index 54a29f8428e9..3eb0bf84208f 100644 --- a/clang/include/clang/Basic/OpenACCKinds.h +++ b/clang/include/clang/Basic/OpenACCKinds.h @@ -90,9 +90,21 @@ enum class OpenACCClauseKind { Vector, /// 'nohost' clause, allowed on 'routine' directives. NoHost, + /// 'default' clause, allowed on parallel, serial, kernel (and compound) + /// constructs. + Default, /// Represents an invalid clause, for the purposes of parsing. Invalid, }; + +enum class OpenACCDefaultClauseKind { + /// 'none' option. + None, + /// 'present' option. + Present, + /// Not a valid option. + Invalid, +}; } // namespace clang #endif // LLVM_CLANG_BASIC_OPENACCKINDS_H diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp index 67325f0a286a..94c3d0c4e164 100644 --- a/clang/lib/Parse/ParseOpenACC.cpp +++ b/clang/lib/Parse/ParseOpenACC.cpp @@ -76,12 +76,17 @@ OpenACCClauseKind getOpenACCClauseKind(Token Tok) { if (Tok.is(tok::kw_auto)) return OpenACCClauseKind::Auto; + // default is a keyword, so make sure we parse it correctly. + if (Tok.is(tok::kw_default)) + return OpenACCClauseKind::Default; + if (!Tok.is(tok::identifier)) return OpenACCClauseKind::Invalid; return llvm::StringSwitch( Tok.getIdentifierInfo()->getName()) .Case("auto", OpenACCClauseKind::Auto) + .Case("default", OpenACCClauseKind::Default) .Case("finalize", OpenACCClauseKind::Finalize) .Case("if_present", OpenACCClauseKind::IfPresent) .Case("independent", OpenACCClauseKind::Independent) @@ -106,6 +111,17 @@ OpenACCAtomicKind getOpenACCAtomicKind(Token Tok) { .Default(OpenACCAtomicKind::Invalid); } +OpenACCDefaultClauseKind getOpenACCDefaultClauseKind(Token Tok) { + if (!Tok.is(tok::identifier)) + return OpenACCDefaultClauseKind::Invalid; + + return llvm::StringSwitch( + Tok.getIdentifierInfo()->getName()) + .Case("none", OpenACCDefaultClauseKind::None) + .Case("present", OpenACCDefaultClauseKind::Present) + .Default(OpenACCDefaultClauseKind::Invalid); +} + enum class OpenACCSpecialTokenKind { ReadOnly, DevNum, @@ -176,6 +192,22 @@ bool isOpenACCDirectiveKind(OpenACCDirectiveKind Kind, Token Tok) { llvm_unreachable("Unknown 'Kind' Passed"); } +/// Used for cases where we expect an identifier-like token, but don't want to +/// give awkward error messages in cases where it is accidentially a keyword. +bool expectIdentifierOrKeyword(Parser &P) { + Token Tok = P.getCurToken(); + + if (Tok.is(tok::identifier)) + return false; + + if (!Tok.isAnnotation() && Tok.getIdentifierInfo() && + Tok.getIdentifierInfo()->isKeyword(P.getLangOpts())) + return false; + + P.Diag(P.getCurToken(), diag::err_expected) << tok::identifier; + return true; +} + OpenACCDirectiveKind ParseOpenACCEnterExitDataDirective(Parser &P, Token FirstTok, OpenACCDirectiveKindEx ExtDirKind) { @@ -291,13 +323,56 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) { return DirKind; } +bool ClauseHasRequiredParens(OpenACCClauseKind Kind) { + return Kind == OpenACCClauseKind::Default; +} + +bool ParseOpenACCClauseParams(Parser &P, OpenACCClauseKind Kind) { + BalancedDelimiterTracker Parens(P, tok::l_paren, + tok::annot_pragma_openacc_end); + + if (ClauseHasRequiredParens(Kind)) { + if (Parens.expectAndConsume()) { + // We are missing a paren, so assume that the person just forgot the + // parameter. Return 'false' so we try to continue on and parse the next + // clause. + P.SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openacc_end, + Parser::StopBeforeMatch); + return false; + } + + switch (Kind) { + case OpenACCClauseKind::Default: { + Token DefKindTok = P.getCurToken(); + + if (expectIdentifierOrKeyword(P)) + break; + + P.ConsumeToken(); + + if (getOpenACCDefaultClauseKind(DefKindTok) == + OpenACCDefaultClauseKind::Invalid) + P.Diag(DefKindTok, diag::err_acc_invalid_default_clause_kind); + + break; + } + default: + llvm_unreachable("Not a required parens type?"); + } + + return Parens.consumeClose(); + } + // FIXME: Handle optional parens + return false; +} + // The OpenACC Clause List is a comma or space-delimited list of clauses (see // the comment on ParseOpenACCClauseList). The concept of a 'clause' doesn't // really have its owner grammar and each individual one has its own definition. -// However, they all are named with a single-identifier (or auto!) token, -// followed in some cases by either braces or parens. +// However, they all are named with a single-identifier (or auto/default!) +// token, followed in some cases by either braces or parens. bool ParseOpenACCClause(Parser &P) { - if (!P.getCurToken().isOneOf(tok::identifier, tok::kw_auto)) + if (!P.getCurToken().isOneOf(tok::identifier, tok::kw_auto, tok::kw_default)) return P.Diag(P.getCurToken(), diag::err_expected) << tok::identifier; OpenACCClauseKind Kind = getOpenACCClauseKind(P.getCurToken()); @@ -309,8 +384,7 @@ bool ParseOpenACCClause(Parser &P) { // Consume the clause name. P.ConsumeToken(); - // FIXME: For future clauses, we need to handle parens/etc below. - return false; + return ParseOpenACCClauseParams(P, Kind); } // Skip until we see the end of pragma token, but don't consume it. This is us diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c index 1e05d82906ae..aedf0c711ad1 100644 --- a/clang/test/ParserOpenACC/parse-clauses.c +++ b/clang/test/ParserOpenACC/parse-clauses.c @@ -54,6 +54,102 @@ void func() { // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} #pragma acc loop seq, +} + +void DefaultClause() { + // expected-error@+2{{expected '('}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial loop default + for(;;){} + + // expected-error@+2{{expected '('}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default seq + for(;;){} + + // expected-error@+2{{expected '('}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default, seq + for(;;){} + + // expected-error@+4{{expected identifier}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default( + for(;;){} + + // expected-error@+4{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default( seq + for(;;){} + + // expected-error@+4{{expected identifier}} + // expected-error@+3{{expected ')'}} + // expected-note@+2{{to match this '('}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default(, seq + for(;;){} + + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default) + for(;;){} + + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default) seq + for(;;){} + + // expected-error@+3{{expected '('}} + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default), seq + for(;;){} + + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default() + for(;;){} + + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default() seq + for(;;){} + + // expected-error@+2{{expected identifier}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default(), seq + for(;;){} + + // expected-error@+2{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default(invalid) + for(;;){} + + // expected-error@+2{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default(auto) seq + for(;;){} + + // expected-error@+2{{invalid value for 'default' clause; expected 'present' or 'none'}} + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default(invalid), seq + for(;;){} + + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default(none) + for(;;){} + + // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} +#pragma acc serial default(present), seq + for(;;){} + + } // expected-warning@+1{{OpenACC directives not yet implemented, pragma ignored}} -- GitLab From c2f5e435a561cb0ebd3334a1377ebbefe8dcf5c3 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 5 Jan 2024 11:44:22 -0600 Subject: [PATCH 138/728] [ELF] Again attempt to fix test on BE architectures Summary: This formats something according to the style, and again attempts to fix this failing on the BE PPC test. Sorry for the spam, these commits are the only way I can check it because the failure isn't local. --- llvm/include/llvm/Object/ELFObjectFile.h | 3 +-- llvm/unittests/Object/ELFObjectFileTest.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index da72b0e335b9..86f42654f8af 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -1353,8 +1353,7 @@ template Triple::ArchType ELFObjectFile::getArch() const { case ELF::EM_CUDA: { if (EF.getHeader().e_ident[ELF::EI_CLASS] == ELF::ELFCLASS32) return Triple::nvptx; - else - return Triple::nvptx64; + return Triple::nvptx64; } case ELF::EM_BPF: diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp index ccf52ae48e92..71c08f51f863 100644 --- a/llvm/unittests/Object/ELFObjectFileTest.cpp +++ b/llvm/unittests/Object/ELFObjectFileTest.cpp @@ -310,7 +310,13 @@ TEST(ELFObjectFileTest, CheckOSAndTriple) { ASSERT_THAT_EXPECTED(ELFObjOrErr, Succeeded()); auto &ELFObj = *ELFObjOrErr; - EXPECT_EQ(Triple, ELFObj.makeTriple().getTriple()); + llvm::Triple TheTriple = ELFObj.makeTriple(); + + // The AMDGPU architecture will be unknown on big-endian testers. + if (TheTriple.getOS() == Triple::UnknownOS) + continue; + + EXPECT_EQ(Triple, TheTriple.getTriple()); } } -- GitLab From 070ac1dcd5cf27ce3722e60c8416217a6fdb5581 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 5 Jan 2024 17:59:57 +0000 Subject: [PATCH 139/728] [SystemZ] vec-perm-14.ll - partially regenerate checks so we can see all the vperm codegen We can't use the script as we need to keep the shuffle mask constant pool checks, but do more than just check that a second vperm isn't generated --- llvm/test/CodeGen/SystemZ/vec-perm-14.ll | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/SystemZ/vec-perm-14.ll b/llvm/test/CodeGen/SystemZ/vec-perm-14.ll index 551ee44c314d..fb3ece96017b 100644 --- a/llvm/test/CodeGen/SystemZ/vec-perm-14.ll +++ b/llvm/test/CodeGen/SystemZ/vec-perm-14.ll @@ -4,8 +4,23 @@ define void @fun() { ; CHECK-LABEL: fun: -; CHECK: vperm -; CHECK-NOT: vperm +; CHECK: # %bb.0: # %bb +; CHECK-NEXT: vlrepf %v0, 0(%r1) +; CHECK-NEXT: vgbm %v1, 0 +; CHECK-NEXT: larl %r1, .LCPI0_0 +; CHECK-NEXT: vceqb %v0, %v0, %v1 +; CHECK-NEXT: vl %v1, 0(%r1), 3 +; CHECK-NEXT: vperm %v0, %v0, %v0, %v1 +; CHECK-NEXT: vlgvf %r0, %v0, 0 +; CHECK-NEXT: tmll %r0, 1 +; CHECK-NEXT: je .LBB0_2 +; CHECK-NEXT: # %bb.1: # %bb1 +; CHECK-NEXT: .LBB0_2: # %bb2 +; CHECK-NEXT: vlgvf %r0, %v0, 1 +; CHECK-NEXT: tmll %r0, 1 +; CHECK-NEXT: je .LBB0_4 +; CHECK-NEXT: # %bb.3: # %bb3 +; CHECK-NEXT: .LBB0_4: # %bb4 bb: %tmp = load <4 x i8>, ptr undef %tmp1 = icmp eq <4 x i8> zeroinitializer, %tmp -- GitLab From 0cb98167ecee4b8e91fb07cb186daae616045454 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 5 Jan 2024 12:09:17 -0600 Subject: [PATCH 140/728] [ELF][Obvious] Last time fixing this test hopefully --- llvm/unittests/Object/ELFObjectFileTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp index 71c08f51f863..48bebc54ccc2 100644 --- a/llvm/unittests/Object/ELFObjectFileTest.cpp +++ b/llvm/unittests/Object/ELFObjectFileTest.cpp @@ -313,7 +313,7 @@ TEST(ELFObjectFileTest, CheckOSAndTriple) { llvm::Triple TheTriple = ELFObj.makeTriple(); // The AMDGPU architecture will be unknown on big-endian testers. - if (TheTriple.getOS() == Triple::UnknownOS) + if (TheTriple.getArch() == Triple::UnknownArch) continue; EXPECT_EQ(Triple, TheTriple.getTriple()); -- GitLab From 848d7af956442fbf08310cc2d094035802fbe6ea Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 6 Jan 2024 02:32:57 +0800 Subject: [PATCH 141/728] [CVP] Improve the value solving of select at use (#76700) This patch improves the value solving of select at use if the condition is an icmp and we know the result of comparison from `LVI->getPredicateAt`. Compile-time impact: http://llvm-compile-time-tracker.com/compare.php?from=7e405eb722e40c79b7726201d0f76b5dab34ba0f&to=3c315b1ddcb0ad82554b33f08b9356679fae4bb7&stat=instructions:u |stage1-O3|stage1-ReleaseThinLTO|stage1-ReleaseLTO-g|stage1-O0-g|stage2-O3|stage2-O0-g|stage2-clang| |--|--|--|--|--|--|--| |-0.01%|+0.01%|-0.00%|-0.00%|-0.08%|+0.02%|-0.01%| --- .../Scalar/CorrelatedValuePropagation.cpp | 51 ++++++++++--------- .../CorrelatedValuePropagation/select.ll | 23 +++++++++ 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp index c44d3748a80d..9235850de92f 100644 --- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp +++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @@ -94,6 +94,31 @@ STATISTIC(NumUDivURemsNarrowedExpanded, "Number of bound udiv's/urem's expanded"); STATISTIC(NumZExt, "Number of non-negative deductions"); +static Constant *getConstantAt(Value *V, Instruction *At, LazyValueInfo *LVI) { + if (Constant *C = LVI->getConstant(V, At)) + return C; + + // TODO: The following really should be sunk inside LVI's core algorithm, or + // at least the outer shims around such. + auto *C = dyn_cast(V); + if (!C) + return nullptr; + + Value *Op0 = C->getOperand(0); + Constant *Op1 = dyn_cast(C->getOperand(1)); + if (!Op1) + return nullptr; + + LazyValueInfo::Tristate Result = LVI->getPredicateAt( + C->getPredicate(), Op0, Op1, At, /*UseBlockValue=*/false); + if (Result == LazyValueInfo::Unknown) + return nullptr; + + return (Result == LazyValueInfo::True) + ? ConstantInt::getTrue(C->getContext()) + : ConstantInt::getFalse(C->getContext()); +} + static bool processSelect(SelectInst *S, LazyValueInfo *LVI) { if (S->getType()->isVectorTy() || isa(S->getCondition())) return false; @@ -106,7 +131,7 @@ static bool processSelect(SelectInst *S, LazyValueInfo *LVI) { C = LVI->getConstantOnEdge(S->getCondition(), PN->getIncomingBlock(U), I->getParent(), I); else - C = LVI->getConstant(S->getCondition(), I); + C = getConstantAt(S->getCondition(), I, LVI); auto *CI = dyn_cast_or_null(C); if (!CI) @@ -1109,30 +1134,6 @@ static bool processAnd(BinaryOperator *BinOp, LazyValueInfo *LVI) { return true; } - -static Constant *getConstantAt(Value *V, Instruction *At, LazyValueInfo *LVI) { - if (Constant *C = LVI->getConstant(V, At)) - return C; - - // TODO: The following really should be sunk inside LVI's core algorithm, or - // at least the outer shims around such. - auto *C = dyn_cast(V); - if (!C) return nullptr; - - Value *Op0 = C->getOperand(0); - Constant *Op1 = dyn_cast(C->getOperand(1)); - if (!Op1) return nullptr; - - LazyValueInfo::Tristate Result = LVI->getPredicateAt( - C->getPredicate(), Op0, Op1, At, /*UseBlockValue=*/false); - if (Result == LazyValueInfo::Unknown) - return nullptr; - - return (Result == LazyValueInfo::True) ? - ConstantInt::getTrue(C->getContext()) : - ConstantInt::getFalse(C->getContext()); -} - static bool runImpl(Function &F, LazyValueInfo *LVI, DominatorTree *DT, const SimplifyQuery &SQ) { bool FnChanged = false; diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/select.ll b/llvm/test/Transforms/CorrelatedValuePropagation/select.ll index 28a8516a9102..9842328db702 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/select.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/select.ll @@ -372,4 +372,27 @@ define i64 @select_cond_may_undef(i32 %a) { ret i64 %max } +define i32 @test_solve_select_at_use(i32 %a, i32 %b, i32 %c) { +; CHECK-LABEL: define i32 @test_solve_select_at_use +; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[A]], 0 +; CHECK-NEXT: [[COND:%.*]] = icmp sgt i32 [[A]], -1 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] +; CHECK: if.then: +; CHECK-NEXT: ret i32 [[C]] +; CHECK: if.else: +; CHECK-NEXT: ret i32 [[B]] +; +entry: + %cmp = icmp slt i32 %a, 0 + %retval = select i1 %cmp, i32 %b, i32 %c + %cond = icmp sgt i32 %a, -1 + br i1 %cond, label %if.then, label %if.else +if.then: + ret i32 %retval +if.else: + ret i32 %retval +} + !0 = !{} -- GitLab From 02a33b72fd21cdbf476d6bda72faa462e073e510 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Fri, 5 Jan 2024 19:43:10 +0100 Subject: [PATCH 142/728] =?UTF-8?q?"Reapply=20"[Sema]=20Fix=20crash=20on?= =?UTF-8?q?=20invalid=20code=20with=20parenthesized=20aggrega=E2=80=A6=20(?= =?UTF-8?q?#76833)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …te initialization" (#76272)"" With updates the libc++ tests. This reverts commit 2205d23 and relands 86dc6e1 and 7ab16fb. Original commit was reverted because of failing libc++ tests, see #76232 for the discussion. The errors in the tests are spurious in the first place (coming from initialization of invalid classes), so update the tests to match new behavior that does not show those errors. The original patch was written by @ilya-biryukov To fix the CI two libc++ tests are temporary disabled for clang-18. --- clang/lib/Sema/SemaInit.cpp | 8 ++++++ clang/test/SemaCXX/crash-GH76228.cpp | 28 +++++++++++++++++++ clang/test/SemaCXX/paren-list-agg-init.cpp | 2 +- .../transform_error.mandates.verify.cpp | 9 ++++++ .../transform_error.mandates.verify.cpp | 9 ++++++ 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 clang/test/SemaCXX/crash-GH76228.cpp diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 60c0e3e74204..e469e420f14f 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -5512,6 +5512,14 @@ static void TryOrBuildParenListInitialization( } else if (auto *RT = Entity.getType()->getAs()) { bool IsUnion = RT->isUnionType(); const CXXRecordDecl *RD = cast(RT->getDecl()); + if (RD->isInvalidDecl()) { + // Exit early to avoid confusion when processing members. + // We do the same for braced list initialization in + // `CheckStructUnionTypes`. + Sequence.SetFailed( + clang::InitializationSequence::FK_ParenthesizedListInitFailed); + return; + } if (!IsUnion) { for (const CXXBaseSpecifier &Base : RD->bases()) { diff --git a/clang/test/SemaCXX/crash-GH76228.cpp b/clang/test/SemaCXX/crash-GH76228.cpp new file mode 100644 index 000000000000..33a939582312 --- /dev/null +++ b/clang/test/SemaCXX/crash-GH76228.cpp @@ -0,0 +1,28 @@ +// RUN: %clang_cc1 -std=c++20 -verify %s +// Check we don't crash on incomplete members and bases when handling parenthesized initialization. +class incomplete; // expected-note@-0 3 {{forward declaration of 'incomplete'}} +struct foo { + int a; + incomplete b; + // expected-error@-1 {{incomplete type}} +}; +foo a1(0); + +struct one_int { + int a; +}; +struct bar : one_int, incomplete {}; +// expected-error@-1 {{incomplete type}} +bar a2(0); + +incomplete a3[3](1,2,3); +// expected-error@-1 {{incomplete type}} + +struct qux : foo { +}; +qux a4(0); + +struct fred { + foo a[3]; +}; +fred a5(0); diff --git a/clang/test/SemaCXX/paren-list-agg-init.cpp b/clang/test/SemaCXX/paren-list-agg-init.cpp index f60b20e0d465..c1964a5a9eb0 100644 --- a/clang/test/SemaCXX/paren-list-agg-init.cpp +++ b/clang/test/SemaCXX/paren-list-agg-init.cpp @@ -289,7 +289,7 @@ int test() { // used to crash S a(0, 1); S b(0); - S c(0, 0, 1); // beforecxx20-warning {{aggregate initialization of type 'S' from a parenthesized list of values is a C++20 extension}} + S c(0, 0, 1); S d {0, 1}; S e {0}; diff --git a/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp index 965e82a7b403..318435660c36 100644 --- a/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp +++ b/libcxx/test/libcxx/utilities/expected/expected.expected/transform_error.mandates.verify.cpp @@ -6,6 +6,15 @@ // //===----------------------------------------------------------------------===// +// https://github.com/llvm/llvm-project/pull/76232 breaks this libc++ test. +// The fix would be to update this file. The issue is that the CI uses 2 +// versions of Clang-18 +// - An older nightly build as the main compiler +// - A freshly bootstrap build +// This means the test can't be used until the nightly build is updated. +// TODO(mordante) Reenable clang-18. +// UNSUPPORTED: clang-18 + // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20 // Test the mandates diff --git a/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp b/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp index 09aa1332e980..d9f65e9f5919 100644 --- a/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp +++ b/libcxx/test/libcxx/utilities/expected/expected.void/transform_error.mandates.verify.cpp @@ -6,6 +6,15 @@ // //===----------------------------------------------------------------------===// +// https://github.com/llvm/llvm-project/pull/76232 breaks this libc++ test. +// The fix would be to update this file. The issue is that the CI uses 2 +// versions of Clang-18 +// - An older nightly build as the main compiler +// - A freshly bootstrap build +// This means the test can't be used until the nightly build is updated. +// TODO(mordante) Reenable clang-18. +// UNSUPPORTED: clang-18 + // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20 // Test the mandates -- GitLab From 31626dadcef86e48bcc44e86e4411578094c8739 Mon Sep 17 00:00:00 2001 From: Nathan Sidwell Date: Fri, 5 Jan 2024 13:50:44 -0500 Subject: [PATCH 143/728] [llvm][NFC] Refactor AutoUpgrader arm/aarch64 (#74145) Break out and refactor AArch64 & ARM intrinsic updating. There's a fair amount of comonality, but let's avoid continually checking the same prefixes. --- llvm/lib/IR/AutoUpgrade.cpp | 499 ++++++++++-------- .../Assembler/autoupgrade-thread-pointer.ll | 4 +- 2 files changed, 284 insertions(+), 219 deletions(-) diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 6b54047020a0..3a3b41fb786c 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -621,6 +621,284 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, return false; } +// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so. +// IsArm: 'arm.*', !IsArm: 'aarch64.*'. +static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, + StringRef Name, + Function *&NewFn) { + if (Name.starts_with("rbit")) { + // '(arm|aarch64).rbit'. + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, + F->arg_begin()->getType()); + return true; + } + + if (Name == "thread.pointer") { + // '(arm|aarch64).thread.pointer'. + NewFn = + Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); + return true; + } + + bool Neon = Name.consume_front("neon."); + if (Neon) { + // '(arm|aarch64).neon.*'. + // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and + // v16i8 respectively. + if (Name.consume_front("bfdot.")) { + // (arm|aarch64).neon.bfdot.*'. + Intrinsic::ID ID = StringSwitch(Name) + .Cases("v2f32.v8i8", "v4f32.v16i8", + IsArm ? Intrinsic::arm_neon_bfdot + : Intrinsic::aarch64_neon_bfdot) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) { + size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits(); + assert((OperandWidth == 64 || OperandWidth == 128) && + "Unexpected operand width"); + LLVMContext &Ctx = F->getParent()->getContext(); + std::array Tys{ + {F->getReturnType(), + FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}}; + NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); + return true; + } + return false; // No other '(arm|aarch64).neon.bfdot.*'. + } + + // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic + // anymore and accept v8bf16 instead of v16i8. + if (Name.consume_front("bfm")) { + // (arm|aarch64).neon.bfm*'. + if (Name.consume_back(".v4f32.v16i8")) { + // (arm|aarch64).neon.bfm*.v4f32.v16i8'. + Intrinsic::ID ID = + StringSwitch(Name) + .Case("mla", IsArm ? Intrinsic::arm_neon_bfmmla + : Intrinsic::aarch64_neon_bfmmla) + .Case("lalb", IsArm ? Intrinsic::arm_neon_bfmlalb + : Intrinsic::aarch64_neon_bfmlalb) + .Case("lalt", IsArm ? Intrinsic::arm_neon_bfmlalt + : Intrinsic::aarch64_neon_bfmlalt) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) { + NewFn = Intrinsic::getDeclaration(F->getParent(), ID); + return true; + } + return false; // No other '(arm|aarch64).neon.bfm*.v16i8'. + } + return false; // No other '(arm|aarch64).neon.bfm*. + } + // Continue on to Aarch64 Neon or Arm Neon. + } + // Continue on to Arm or Aarch64. + + if (IsArm) { + // 'arm.*'. + if (Neon) { + // 'arm.neon.*'. + Intrinsic::ID ID = StringSwitch(Name) + .StartsWith("vclz.", Intrinsic::ctlz) + .StartsWith("vcnt.", Intrinsic::ctpop) + .StartsWith("vqadds.", Intrinsic::sadd_sat) + .StartsWith("vqaddu.", Intrinsic::uadd_sat) + .StartsWith("vqsubs.", Intrinsic::ssub_sat) + .StartsWith("vqsubu.", Intrinsic::usub_sat) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) { + NewFn = Intrinsic::getDeclaration(F->getParent(), ID, + F->arg_begin()->getType()); + return true; + } + + if (Name.consume_front("vst")) { + // 'arm.neon.vst*'. + static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$"); + SmallVector Groups; + if (vstRegex.match(Name, &Groups)) { + static const Intrinsic::ID StoreInts[] = { + Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2, + Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4}; + + static const Intrinsic::ID StoreLaneInts[] = { + Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane, + Intrinsic::arm_neon_vst4lane}; + + auto fArgs = F->getFunctionType()->params(); + Type *Tys[] = {fArgs[0], fArgs[1]}; + if (Groups[1].size() == 1) + NewFn = Intrinsic::getDeclaration(F->getParent(), + StoreInts[fArgs.size() - 3], Tys); + else + NewFn = Intrinsic::getDeclaration( + F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys); + return true; + } + return false; // No other 'arm.neon.vst*'. + } + + return false; // No other 'arm.neon.*'. + } + + if (Name.consume_front("mve.")) { + // 'arm.mve.*'. + if (Name == "vctp64") { + if (cast(F->getReturnType())->getNumElements() == 4) { + // A vctp64 returning a v4i1 is converted to return a v2i1. Rename + // the function and deal with it below in UpgradeIntrinsicCall. + rename(F); + return true; + } + return false; // Not 'arm.mve.vctp64'. + } + + // These too are changed to accept a v2i1 instead of the old v4i1. + if (Name.consume_back(".v4i1")) { + // 'arm.mve.*.v4i1'. + if (Name.consume_back(".predicated.v2i64.v4i32")) + // 'arm.mve.*.predicated.v2i64.v4i32.v4i1' + return Name == "mull.int" || Name == "vqdmull"; + + if (Name.consume_back(".v2i64")) { + // 'arm.mve.*.v2i64.v4i1' + bool IsGather = Name.consume_front("vldr.gather."); + if (IsGather || Name.consume_front("vstr.scatter.")) { + if (Name.consume_front("base.")) { + // Optional 'wb.' prefix. + Name.consume_front("wb."); + // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)? + // predicated.v2i64.v2i64.v4i1'. + return Name == "predicated.v2i64"; + } + + if (Name.consume_front("offset.predicated.")) + return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") || + Name == (IsGather ? "v2i64.p0" : "p0.v2i64"); + + // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'. + return false; + } + + return false; // No other 'arm.mve.*.v2i64.v4i1'. + } + return false; // No other 'arm.mve.*.v4i1'. + } + return false; // No other 'arm.mve.*'. + } + + if (Name.consume_front("cde.vcx")) { + // 'arm.cde.vcx*'. + if (Name.consume_back(".predicated.v2i64.v4i1")) + // 'arm.cde.vcx*.predicated.v2i64.v4i1'. + return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" || + Name == "3q" || Name == "3qa"; + + return false; // No other 'arm.cde.vcx*'. + } + } else { + // 'aarch64.*'. + if (Neon) { + // 'aarch64.neon.*'. + Intrinsic::ID ID = StringSwitch(Name) + .StartsWith("frintn", Intrinsic::roundeven) + .StartsWith("rbit", Intrinsic::bitreverse) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) { + NewFn = Intrinsic::getDeclaration(F->getParent(), ID, + F->arg_begin()->getType()); + return true; + } + + if (Name.starts_with("addp")) { + // 'aarch64.neon.addp*'. + if (F->arg_size() != 2) + return false; // Invalid IR. + VectorType *Ty = dyn_cast(F->getReturnType()); + if (Ty && Ty->getElementType()->isFloatingPointTy()) { + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::aarch64_neon_faddp, Ty); + return true; + } + } + return false; // No other 'aarch64.neon.*'. + } + if (Name.consume_front("sve.")) { + // 'aarch64.sve.*'. + if (Name.consume_front("bf")) { + if (Name.consume_back(".lane")) { + // 'aarch64.sve.bf*.lane'. + Intrinsic::ID ID = + StringSwitch(Name) + .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2) + .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2) + .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) { + NewFn = Intrinsic::getDeclaration(F->getParent(), ID); + return true; + } + return false; // No other 'aarch64.sve.bf*.lane'. + } + return false; // No other 'aarch64.sve.bf*'. + } + + if (Name.consume_front("ld")) { + // 'aarch64.sve.ld*'. + static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)"); + if (LdRegex.match(Name)) { + Type *ScalarTy = + dyn_cast(F->getReturnType())->getElementType(); + ElementCount EC = dyn_cast(F->arg_begin()->getType()) + ->getElementCount(); + Type *Ty = VectorType::get(ScalarTy, EC); + static const Intrinsic::ID LoadIDs[] = { + Intrinsic::aarch64_sve_ld2_sret, + Intrinsic::aarch64_sve_ld3_sret, + Intrinsic::aarch64_sve_ld4_sret, + }; + NewFn = Intrinsic::getDeclaration(F->getParent(), + LoadIDs[Name[0] - '2'], Ty); + return true; + } + return false; // No other 'aarch64.sve.ld*'. + } + + if (Name.consume_front("tuple.")) { + // 'aarch64.sve.tuple.*'. + if (Name.starts_with("get")) { + // 'aarch64.sve.tuple.get*'. + Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()}; + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::vector_extract, Tys); + return true; + } + + if (Name.starts_with("set")) { + // 'aarch64.sve.tuple.set*'. + auto Args = F->getFunctionType()->params(); + Type *Tys[] = {Args[0], Args[2], Args[1]}; + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::vector_insert, Tys); + return true; + } + + static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)"); + if (CreateTupleRegex.match(Name)) { + // 'aarch64.sve.tuple.create*'. + auto Args = F->getFunctionType()->params(); + Type *Tys[] = {F->getReturnType(), Args[1]}; + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::vector_insert, Tys); + return true; + } + return false; // No other 'aarch64.sve.tuple.*'. + } + return false; // No other 'aarch64.sve.*'. + } + } + return false; // No other 'arm.*', 'aarch64.*'. +} + static Intrinsic::ID ShouldUpgradeNVPTXBF16Intrinsic(StringRef Name) { if (Name.consume_front("abs.")) return StringSwitch(Name) @@ -713,225 +991,12 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { switch (Name[0]) { default: break; case 'a': { - if (Name.starts_with("arm.rbit") || Name.starts_with("aarch64.rbit")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, - F->arg_begin()->getType()); - return true; - } - if (Name.starts_with("aarch64.neon.frintn")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven, - F->arg_begin()->getType()); - return true; - } - if (Name.starts_with("aarch64.neon.rbit")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, - F->arg_begin()->getType()); - return true; - } - if (Name == "aarch64.sve.bfdot.lane") { - NewFn = Intrinsic::getDeclaration(F->getParent(), - Intrinsic::aarch64_sve_bfdot_lane_v2); - return true; - } - if (Name == "aarch64.sve.bfmlalb.lane") { - NewFn = Intrinsic::getDeclaration(F->getParent(), - Intrinsic::aarch64_sve_bfmlalb_lane_v2); - return true; - } - if (Name == "aarch64.sve.bfmlalt.lane") { - NewFn = Intrinsic::getDeclaration(F->getParent(), - Intrinsic::aarch64_sve_bfmlalt_lane_v2); - return true; - } - static const Regex LdRegex("^aarch64\\.sve\\.ld[234](.nxv[a-z0-9]+|$)"); - if (LdRegex.match(Name)) { - Type *ScalarTy = - dyn_cast(F->getReturnType())->getElementType(); - ElementCount EC = - dyn_cast(F->arg_begin()->getType())->getElementCount(); - Type *Ty = VectorType::get(ScalarTy, EC); - Intrinsic::ID ID = - StringSwitch(Name) - .StartsWith("aarch64.sve.ld2", Intrinsic::aarch64_sve_ld2_sret) - .StartsWith("aarch64.sve.ld3", Intrinsic::aarch64_sve_ld3_sret) - .StartsWith("aarch64.sve.ld4", Intrinsic::aarch64_sve_ld4_sret) - .Default(Intrinsic::not_intrinsic); - NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Ty); - return true; - } - if (Name.starts_with("aarch64.sve.tuple.get")) { - Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()}; - NewFn = Intrinsic::getDeclaration(F->getParent(), - Intrinsic::vector_extract, Tys); - return true; - } - if (Name.starts_with("aarch64.sve.tuple.set")) { - auto Args = F->getFunctionType()->params(); - Type *Tys[] = {Args[0], Args[2], Args[1]}; - NewFn = Intrinsic::getDeclaration(F->getParent(), - Intrinsic::vector_insert, Tys); - return true; - } - static const Regex CreateTupleRegex( - "^aarch64\\.sve\\.tuple\\.create[234](.nxv[a-z0-9]+|$)"); - if (CreateTupleRegex.match(Name)) { - auto Args = F->getFunctionType()->params(); - Type *Tys[] = {F->getReturnType(), Args[1]}; - NewFn = Intrinsic::getDeclaration(F->getParent(), - Intrinsic::vector_insert, Tys); - return true; - } - if (Name.starts_with("arm.neon.vclz")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, - F->arg_begin()->getType()); - return true; - } - if (Name.starts_with("arm.neon.vcnt")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, - F->arg_begin()->getType()); - return true; - } - static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$"); - if (vstRegex.match(Name)) { - static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1, - Intrinsic::arm_neon_vst2, - Intrinsic::arm_neon_vst3, - Intrinsic::arm_neon_vst4}; - - static const Intrinsic::ID StoreLaneInts[] = { - Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane, - Intrinsic::arm_neon_vst4lane - }; - - auto fArgs = F->getFunctionType()->params(); - Type *Tys[] = {fArgs[0], fArgs[1]}; - if (!Name.contains("lane")) - NewFn = Intrinsic::getDeclaration(F->getParent(), - StoreInts[fArgs.size() - 3], Tys); - else - NewFn = Intrinsic::getDeclaration(F->getParent(), - StoreLaneInts[fArgs.size() - 5], Tys); - return true; - } - if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); - return true; - } - if (Name.starts_with("arm.neon.vqadds.")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat, - F->arg_begin()->getType()); - return true; - } - if (Name.starts_with("arm.neon.vqaddu.")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat, - F->arg_begin()->getType()); - return true; - } - if (Name.starts_with("arm.neon.vqsubs.")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat, - F->arg_begin()->getType()); - return true; - } - if (Name.starts_with("arm.neon.vqsubu.")) { - NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat, - F->arg_begin()->getType()); - return true; - } - if (Name.starts_with("aarch64.neon.addp")) { - if (F->arg_size() != 2) - break; // Invalid IR. - VectorType *Ty = dyn_cast(F->getReturnType()); - if (Ty && Ty->getElementType()->isFloatingPointTy()) { - NewFn = Intrinsic::getDeclaration(F->getParent(), - Intrinsic::aarch64_neon_faddp, Ty); + bool IsArm = Name.consume_front("arm."); + if (IsArm || Name.consume_front("aarch64.")) { + if (UpgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn)) return true; - } - } - - // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8 - // respectively - if ((Name.starts_with("arm.neon.bfdot.") || - Name.starts_with("aarch64.neon.bfdot.")) && - Name.ends_with("i8")) { - Intrinsic::ID IID = - StringSwitch(Name) - .Cases("arm.neon.bfdot.v2f32.v8i8", - "arm.neon.bfdot.v4f32.v16i8", - Intrinsic::arm_neon_bfdot) - .Cases("aarch64.neon.bfdot.v2f32.v8i8", - "aarch64.neon.bfdot.v4f32.v16i8", - Intrinsic::aarch64_neon_bfdot) - .Default(Intrinsic::not_intrinsic); - if (IID == Intrinsic::not_intrinsic) - break; - - size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits(); - assert((OperandWidth == 64 || OperandWidth == 128) && - "Unexpected operand width"); - LLVMContext &Ctx = F->getParent()->getContext(); - std::array Tys {{ - F->getReturnType(), - FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16) - }}; - NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys); - return true; - } - - // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore - // and accept v8bf16 instead of v16i8 - if ((Name.starts_with("arm.neon.bfm") || - Name.starts_with("aarch64.neon.bfm")) && - Name.ends_with(".v4f32.v16i8")) { - Intrinsic::ID IID = - StringSwitch(Name) - .Case("arm.neon.bfmmla.v4f32.v16i8", - Intrinsic::arm_neon_bfmmla) - .Case("arm.neon.bfmlalb.v4f32.v16i8", - Intrinsic::arm_neon_bfmlalb) - .Case("arm.neon.bfmlalt.v4f32.v16i8", - Intrinsic::arm_neon_bfmlalt) - .Case("aarch64.neon.bfmmla.v4f32.v16i8", - Intrinsic::aarch64_neon_bfmmla) - .Case("aarch64.neon.bfmlalb.v4f32.v16i8", - Intrinsic::aarch64_neon_bfmlalb) - .Case("aarch64.neon.bfmlalt.v4f32.v16i8", - Intrinsic::aarch64_neon_bfmlalt) - .Default(Intrinsic::not_intrinsic); - if (IID == Intrinsic::not_intrinsic) - break; - - std::array Tys; - NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys); - return true; - } - - if (Name == "arm.mve.vctp64" && - cast(F->getReturnType())->getNumElements() == 4) { - // A vctp64 returning a v4i1 is converted to return a v2i1. Rename the - // function and deal with it below in UpgradeIntrinsicCall. - rename(F); - return true; + break; } - // These too are changed to accept a v2i1 insteead of the old v4i1. - if (Name == "arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" || - Name == "arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" || - Name == "arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" || - Name == "arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" || - Name == - "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" || - Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" || - Name == "arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" || - Name == "arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" || - Name == - "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" || - Name == "arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" || - Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" || - Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" || - Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" || - Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" || - Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" || - Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1") - return true; if (Name.consume_front("amdgcn.")) { if (Name == "alignbit") { diff --git a/llvm/test/Assembler/autoupgrade-thread-pointer.ll b/llvm/test/Assembler/autoupgrade-thread-pointer.ll index 2eaabb4e921f..b1ed15a7e4ef 100644 --- a/llvm/test/Assembler/autoupgrade-thread-pointer.ll +++ b/llvm/test/Assembler/autoupgrade-thread-pointer.ll @@ -5,14 +5,14 @@ declare ptr @llvm.aarch64.thread.pointer() declare ptr @llvm.arm.thread.pointer() define ptr @test1() { -; CHECK: test1() +; CHECK-LABEL: define ptr @test1() ; CHECK: call ptr @llvm.thread.pointer() %1 = call ptr @llvm.aarch64.thread.pointer() ret ptr %1 } define ptr @test2() { -; CHECK: test2() +; CHECK-LABEL: define ptr @test2() ; CHECK: call ptr @llvm.thread.pointer() %1 = call ptr @llvm.arm.thread.pointer() ret ptr %1 -- GitLab From 8d6784db04ee5d925a2d036a68f00a7c124c6cf9 Mon Sep 17 00:00:00 2001 From: Jordan Rupprecht Date: Fri, 5 Jan 2024 11:07:42 -0800 Subject: [PATCH 144/728] [bazel][libc][math] Add missing nan(f|l) deps After 0504e932880e0c4c333baeba9b8d674c5b9ae316 --- .../llvm-project-overlay/libc/BUILD.bazel | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 7d2bb2980255..f222831eefd7 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -540,6 +540,7 @@ libc_support_library( ":__support_cpp_bit", ":__support_cpp_limits", ":__support_cpp_optional", + ":__support_cpp_string_view", ":__support_ctype_utils", ":__support_fputil_dyadic_float", ":__support_fputil_fenv_impl", @@ -1874,11 +1875,29 @@ libc_math_function(name = "llroundf") libc_math_function(name = "llroundl") -libc_math_function(name = "nan") +libc_math_function( + name = "nan", + additional_deps = [ + ":__support_str_to_float", + ":errno", + ], +) -libc_math_function(name = "nanf") +libc_math_function( + name = "nanf", + additional_deps = [ + ":__support_str_to_float", + ":errno", + ], +) -libc_math_function(name = "nanl") +libc_math_function( + name = "nanl", + additional_deps = [ + ":__support_str_to_float", + ":errno", + ], +) libc_math_function(name = "nearbyint") -- GitLab From 93d2e49b6aa6904620a7e9b559c04ec8756dc596 Mon Sep 17 00:00:00 2001 From: Shubham Sandeep Rastogi Date: Fri, 5 Jan 2024 11:15:18 -0800 Subject: [PATCH 145/728] Fix file index verifier when there is no file name in the prologue. (#77004) If there is no file name in the prologue of a line table, the verifier will try to verify the file index, which will be set to 1 by default. This will cause the DWARF verifier to throw an error even if there is no error. rdar://114476503 rdar://114343624 --- llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp | 9 +- .../tools/llvm-dwarfdump/verify-no-file.yaml | 356 ++++++++++++++++++ 2 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp index 43ed60d7f977..18377815c78b 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp @@ -940,8 +940,13 @@ void DWARFVerifier::verifyDebugLineRows() { OS << '\n'; } - // Verify file index. - if (!LineTable->hasFileAtIndex(Row.File)) { + // If the prologue contains no file names and the line table has only one + // row, do not verify the file index, this is a line table of an empty + // file with an end_sequence, but the DWARF standard sets the file number + // to 1 by default, otherwise verify file index. + if (!(LineTable->Prologue.FileNames.size() == 0 && + LineTable->Rows.size() == 1) && + !LineTable->hasFileAtIndex(Row.File)) { ++NumDebugLineErrors; error() << ".debug_line[" << format("0x%08" PRIx64, diff --git a/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml b/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml new file mode 100644 index 000000000000..e43bb8808216 --- /dev/null +++ b/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml @@ -0,0 +1,356 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: yaml2obj %s -o %t/test.o +# RUN: llvm-dwarfdump --debug-line --verify %t/test.o 2>&1 | FileCheck %s + +# CHECK-NOT: error: .debug_line[0x{{[0-9a-f]+}}][0] has invalid file index 1 (valid values are [1,0]): +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x100000C + cpusubtype: 0x0 + filetype: 0x1 + ncmds: 4 + sizeofcmds: 1000 + flags: 0x2000 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 872 + segname: '' + vmaddr: 0 + vmsize: 635 + fileoff: 1032 + filesize: 635 + maxprot: 7 + initprot: 7 + nsects: 10 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0x0 + size: 4 + offset: 0x408 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x80000400 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: C0035FD6 + - sectname: __debug_abbrev + segname: __DWARF + addr: 0x4 + size: 50 + offset: 0x40C + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - sectname: __debug_info + segname: __DWARF + addr: 0x36 + size: 76 + offset: 0x43E + align: 0 + reloff: 0x688 + nreloc: 2 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + relocations: + - address: 0x33 + symbolnum: 1 + pcrel: false + length: 3 + extern: false + type: 0 + scattered: false + value: 0 + - address: 0x26 + symbolnum: 1 + pcrel: false + length: 3 + extern: false + type: 0 + scattered: false + value: 0 + - sectname: __debug_str + segname: __DWARF + addr: 0x82 + size: 196 + offset: 0x48A + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - sectname: __apple_names + segname: __DWARF + addr: 0x146 + size: 88 + offset: 0x54E + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 485341480100000002000000020000000C000000000000000100000001000600FFFFFFFF000000008973880BEB28616A3800000048000000B8000000010000003200000000000000BC000000010000003200000000000000 + - sectname: __apple_objc + segname: __DWARF + addr: 0x19E + size: 36 + offset: 0x5A6 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 485341480100000001000000000000000C000000000000000100000001000600FFFFFFFF + - sectname: __apple_namespac + segname: __DWARF + addr: 0x1C2 + size: 36 + offset: 0x5CA + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 485341480100000001000000000000000C000000000000000100000001000600FFFFFFFF + - sectname: __apple_types + segname: __DWARF + addr: 0x1E6 + size: 44 + offset: 0x5EE + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 48534148010000000100000000000000140000000000000003000000010006000300050004000B00FFFFFFFF + - sectname: __compact_unwind + segname: __LD + addr: 0x218 + size: 32 + offset: 0x620 + align: 3 + reloff: 0x698 + nreloc: 1 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '0000000000000000040000000000000200000000000000000000000000000000' + relocations: + - address: 0x0 + symbolnum: 1 + pcrel: false + length: 3 + extern: false + type: 0 + scattered: false + value: 0 + - sectname: __debug_line + segname: __DWARF + addr: 0x238 + size: 67 + offset: 0x640 + align: 0 + reloff: 0x6A0 + nreloc: 1 + flags: 0x2000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + relocations: + - address: 0x35 + symbolnum: 1 + pcrel: false + length: 3 + extern: false + type: 0 + scattered: false + value: 0 + - cmd: LC_BUILD_VERSION + cmdsize: 24 + platform: 1 + minos: 917504 + sdk: 918528 + ntools: 0 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 1704 + nsyms: 3 + stroff: 1752 + strsize: 24 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 2 + iextdefsym: 2 + nextdefsym: 1 + iundefsym: 3 + nundefsym: 0 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 +LinkEditData: + NameList: + - n_strx: 16 + n_type: 0xE + n_sect: 1 + n_desc: 0 + n_value: 0 + - n_strx: 10 + n_type: 0xE + n_sect: 9 + n_desc: 0 + n_value: 536 + - n_strx: 1 + n_type: 0xF + n_sect: 1 + n_desc: 0 + n_value: 0 + StringTable: + - '' + - __Z3foov + - ltmp1 + - ltmp0 + - '' + - '' +DWARF: + debug_str: + - 'Apple clang version 16.0.0 (clang-1600.0.9.14)' + - '/tmp/test.cpp' + - '/Library/Developer/CommandLineTools/SDKs/MacOSX14.4.sdk' + - MacOSX14.4.sdk + - '/Users/shubham/Development/llvm-project/build_ninja' + - foo + - _Z3foov + debug_abbrev: + - ID: 0 + Table: + - Code: 0x1 + Tag: DW_TAG_compile_unit + Children: DW_CHILDREN_yes + Attributes: + - Attribute: DW_AT_producer + Form: DW_FORM_strp + - Attribute: DW_AT_language + Form: DW_FORM_data2 + - Attribute: DW_AT_name + Form: DW_FORM_strp + - Attribute: DW_AT_LLVM_sysroot + Form: DW_FORM_strp + - Attribute: DW_AT_APPLE_sdk + Form: DW_FORM_strp + - Attribute: DW_AT_stmt_list + Form: DW_FORM_sec_offset + - Attribute: DW_AT_comp_dir + Form: DW_FORM_strp + - Attribute: DW_AT_low_pc + Form: DW_FORM_addr + - Attribute: DW_AT_high_pc + Form: DW_FORM_data4 + - Code: 0x2 + Tag: DW_TAG_subprogram + Children: DW_CHILDREN_no + Attributes: + - Attribute: DW_AT_low_pc + Form: DW_FORM_addr + - Attribute: DW_AT_high_pc + Form: DW_FORM_data4 + - Attribute: DW_AT_APPLE_omit_frame_ptr + Form: DW_FORM_flag_present + - Attribute: DW_AT_frame_base + Form: DW_FORM_exprloc + - Attribute: DW_AT_linkage_name + Form: DW_FORM_strp + - Attribute: DW_AT_name + Form: DW_FORM_strp + - Attribute: DW_AT_decl_file + Form: DW_FORM_data1 + - Attribute: DW_AT_decl_line + Form: DW_FORM_data1 + - Attribute: DW_AT_external + Form: DW_FORM_flag_present + debug_info: + - Length: 0x48 + Version: 4 + AbbrevTableID: 0 + AbbrOffset: 0x0 + AddrSize: 8 + Entries: + - AbbrCode: 0x1 + Values: + - Value: 0x0 + - Value: 0x4 + - Value: 0x2F + - Value: 0x3D + - Value: 0x75 + - Value: 0x0 + - Value: 0x84 + - Value: 0x0 + - Value: 0x4 + - AbbrCode: 0x2 + Values: + - Value: 0x0 + - Value: 0x4 + - Value: 0x1 + - Value: 0x1 + BlockData: [ 0x6F ] + - Value: 0xBC + - Value: 0xB8 + - Value: 0x1 + - Value: 0x1 + - Value: 0x1 + - AbbrCode: 0x0 + debug_line: + - Length: 38 + Version: 4 + PrologueLength: 29 + MinInstLength: 1 + MaxOpsPerInst: 1 + DefaultIsStmt: 1 + LineBase: 251 + LineRange: 14 + OpcodeBase: 13 + StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ] + IncludeDirs: + - '/tmp' + Files: + - Name: '' + DirIdx: 10 + ModTime: 11 + Length: 12 + Opcodes: + - Opcode: DW_LNS_extended_op + ExtLen: 1 + SubOpcode: DW_LNE_end_sequence + Data: 0 +... -- GitLab From f9aba820eb6478a617145eb690ba8b06490d7ac5 Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Fri, 5 Jan 2024 14:34:53 -0500 Subject: [PATCH 146/728] [gn] port a0e6b7c04292 --- llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn index 36d5b103e4cf..ac0dd8674355 100644 --- a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn @@ -63,6 +63,7 @@ executable("llvm-tblgen") { "InfoByHwMode.cpp", "InstrDocsEmitter.cpp", "InstrInfoEmitter.cpp", + "MacroFusionPredicatorEmitter.cpp", "OptEmitter.cpp", "OptParserEmitter.cpp", "OptRSTEmitter.cpp", -- GitLab From f22dc88759a53d4224c17d3833a359ef5674b4ea Mon Sep 17 00:00:00 2001 From: Shubham Sandeep Rastogi Date: Fri, 5 Jan 2024 12:13:36 -0800 Subject: [PATCH 147/728] [NFC] Address review feedback from PR #77004 (#77134) Accidentally didn't commit the review feedback before merging the PR --- llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp | 4 ++-- llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp index 18377815c78b..d25b732fdba3 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFVerifier.cpp @@ -944,8 +944,8 @@ void DWARFVerifier::verifyDebugLineRows() { // row, do not verify the file index, this is a line table of an empty // file with an end_sequence, but the DWARF standard sets the file number // to 1 by default, otherwise verify file index. - if (!(LineTable->Prologue.FileNames.size() == 0 && - LineTable->Rows.size() == 1) && + if ((LineTable->Prologue.FileNames.size() || + LineTable->Rows.size() != 1) && !LineTable->hasFileAtIndex(Row.File)) { ++NumDebugLineErrors; error() << ".debug_line[" diff --git a/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml b/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml index e43bb8808216..fc25fceef764 100644 --- a/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml +++ b/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml @@ -1,6 +1,5 @@ -# RUN: rm -rf %t && mkdir -p %t -# RUN: yaml2obj %s -o %t/test.o -# RUN: llvm-dwarfdump --debug-line --verify %t/test.o 2>&1 | FileCheck %s +# RUN: yaml2obj %s -o %t.o +# RUN: llvm-dwarfdump --debug-line --verify %t.o 2>&1 | FileCheck %s # CHECK-NOT: error: .debug_line[0x{{[0-9a-f]+}}][0] has invalid file index 1 (valid values are [1,0]): --- !mach-o -- GitLab From 5121e2cffd23751360e71a8ac69b7462dae23aa8 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 5 Jan 2024 14:29:34 -0600 Subject: [PATCH 148/728] [OpenMP] Change `__tgt_device_image` to point to the image (#77003) Summary: We use the OffloadBinary to contain bundled offloading objects used to support many images / targets at the same time. The `__tgt_device_info` struct used to contain a pointer to this underlying binary format, which contains information about the triple and architecture. We used to parse this in the runtime to do image verification. Recent changes removed the need for this to be used internally, as we just parse it out of the ELF directly. This patch sets the pointers up so they point to the ELF without requiring any further parsing. --- clang/test/Driver/linker-wrapper-image.c | 4 ++-- .../clang-linker-wrapper/OffloadWrapper.cpp | 24 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c index a2a1996f6643..40dde2e02918 100644 --- a/clang/test/Driver/linker-wrapper-image.c +++ b/clang/test/Driver/linker-wrapper-image.c @@ -19,8 +19,8 @@ // OPENMP-COFF: @__start_omp_offloading_entries = hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "omp_offloading_entries$OA" // OPENMP-COFF-NEXT: @__stop_omp_offloading_entries = hidden constant [0 x %struct.__tgt_offload_entry] zeroinitializer, section "omp_offloading_entries$OZ" -// OPENMP: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}" -// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr @.omp_offloading.device_image, ptr getelementptr inbounds ([[[SIZE]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }] +// OPENMP-NEXT: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD\01{{.*}}", section ".llvm.offloading", align 8 +// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr inbounds ([[[BEGIN:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr getelementptr inbounds ([[[END:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 1, i64 0), ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries }] // OPENMP-NEXT: @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_omp_offloading_entries, ptr @__stop_omp_offloading_entries } // OPENMP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.omp_offloading.descriptor_reg, ptr null }] // OPENMP-NEXT: @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @.omp_offloading.descriptor_unreg, ptr null }] diff --git a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp index f4f500b17357..161374ae5552 100644 --- a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp @@ -8,6 +8,7 @@ #include "OffloadWrapper.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/BinaryFormat/Magic.h" #include "llvm/Frontend/Offloading/Utility.h" #include "llvm/IR/Constants.h" #include "llvm/IR/GlobalVariable.h" @@ -121,19 +122,36 @@ GlobalVariable *createBinDesc(Module &M, ArrayRef> Bufs) { SmallVector ImagesInits; ImagesInits.reserve(Bufs.size()); for (ArrayRef Buf : Bufs) { + // We embed the full offloading entry so the binary utilities can parse it. auto *Data = ConstantDataArray::get(C, Buf); - auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant*/ true, + auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true, GlobalVariable::InternalLinkage, Data, ".omp_offloading.device_image"); Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); Image->setSection(".llvm.offloading"); Image->setAlignment(Align(object::OffloadBinary::getAlignment())); - auto *Size = ConstantInt::get(getSizeTTy(M), Buf.size()); + StringRef Binary(Buf.data(), Buf.size()); + assert(identify_magic(Binary) == file_magic::offload_binary && + "Invalid binary format"); + + // The device image struct contains the pointer to the beginning and end of + // the image stored inside of the offload binary. There should only be one + // of these for each buffer so we parse it out manually. + const auto *Header = + reinterpret_cast( + Binary.bytes_begin()); + const auto *Entry = reinterpret_cast( + Binary.bytes_begin() + Header->EntryOffset); + + auto *Begin = ConstantInt::get(getSizeTTy(M), Entry->ImageOffset); + auto *Size = + ConstantInt::get(getSizeTTy(M), Entry->ImageOffset + Entry->ImageSize); + Constant *ZeroBegin[] = {Zero, Begin}; Constant *ZeroSize[] = {Zero, Size}; auto *ImageB = - ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroZero); + ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroBegin); auto *ImageE = ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroSize); -- GitLab From 7c3bcc307a8fa9153a171f6abb4e8fdc91bd6030 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Sat, 6 Jan 2024 04:30:07 +0800 Subject: [PATCH 149/728] [InstCombine] Fold `switch(zext/sext(X))` into `switch(X)` (#76988) This patch folds `switch(zext/sext(X))` into `switch(X)`. The original motivation of this patch is to optimize a pattern found in cvc5. For example: ``` %bf.load.i = load i16, ptr %d_kind.i, align 8 %bf.clear.i = and i16 %bf.load.i, 1023 %bf.cast.i = zext nneg i16 %bf.clear.i to i32 switch i32 %bf.cast.i, label %if.else [ i32 335, label %if.then i32 303, label %if.then ] if.then: ; preds = %entry, %entry %d_children.i.i = getelementptr inbounds %"class.cvc5::internal::expr::NodeValue", ptr %0, i64 0, i32 3 %cmp.i.i.i.i.i = icmp eq i16 %bf.clear.i, 1023 %cond.i.i.i.i.i = select i1 %cmp.i.i.i.i.i, i32 -1, i32 %bf.cast.i ``` `%cmp.i.i.i.i.i` always evaluates to false because `%bf.clear.i` can only be 335 or 303. Folding `switch i32 %bf.cast.i` to `switch i16 %bf.clear.i` will help `CVP` to handle this case. See also https://github.com/llvm/llvm-project/pull/76928#issuecomment-1877055722. Compile-time impact: http://llvm-compile-time-tracker.com/compare.php?from=7954c57124b495fbdc73674d71f2e366e4afe522&to=502b13ed34e561d995ae1f724cf06d20008bd86f&stat=instructions:u |stage1-O3|stage1-ReleaseThinLTO|stage1-ReleaseLTO-g|stage1-O0-g|stage2-O3|stage2-O0-g|stage2-clang| |--|--|--|--|--|--|--| |+0.03%|+0.06%|+0.07%|+0.00%|-0.02%|-0.03%|+0.02%| --- .../InstCombine/InstructionCombining.cpp | 19 +++ llvm/test/Transforms/InstCombine/phi.ll | 30 ++--- .../InstCombine/switch-zext-sext.ll | 122 ++++++++++++++++++ .../Transforms/PhaseOrdering/switch-sext.ll | 48 +++++++ 4 files changed, 202 insertions(+), 17 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/switch-zext-sext.ll create mode 100644 llvm/test/Transforms/PhaseOrdering/switch-sext.ll diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index e845bf38df2d..7f2018b3a199 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -3247,6 +3247,25 @@ Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) { } } + // Fold switch(zext/sext(X)) into switch(X) if possible. + if (match(Cond, m_ZExtOrSExt(m_Value(Op0)))) { + bool IsZExt = isa(Cond); + Type *SrcTy = Op0->getType(); + unsigned NewWidth = SrcTy->getScalarSizeInBits(); + + if (all_of(SI.cases(), [&](const auto &Case) { + const APInt &CaseVal = Case.getCaseValue()->getValue(); + return IsZExt ? CaseVal.isIntN(NewWidth) + : CaseVal.isSignedIntN(NewWidth); + })) { + for (auto &Case : SI.cases()) { + APInt TruncatedCase = Case.getCaseValue()->getValue().trunc(NewWidth); + Case.setValue(ConstantInt::get(SI.getContext(), TruncatedCase)); + } + return replaceOperand(SI, 0, Op0); + } + } + KnownBits Known = computeKnownBits(Cond, 0, &SI); unsigned LeadingKnownZeros = Known.countMinLeadingZeros(); unsigned LeadingKnownOnes = Known.countMinLeadingOnes(); diff --git a/llvm/test/Transforms/InstCombine/phi.ll b/llvm/test/Transforms/InstCombine/phi.ll index 90818771675b..717f4c682a15 100644 --- a/llvm/test/Transforms/InstCombine/phi.ll +++ b/llvm/test/Transforms/InstCombine/phi.ll @@ -996,10 +996,9 @@ done: define i1 @PR24766(i8 %x1, i8 %x2, i8 %condition) { ; CHECK-LABEL: @PR24766( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32 -; CHECK-NEXT: switch i32 [[CONV]], label [[EPILOG:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW1:%.*]] -; CHECK-NEXT: i32 1, label [[SW2:%.*]] +; CHECK-NEXT: switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [ +; CHECK-NEXT: i8 0, label [[SW1:%.*]] +; CHECK-NEXT: i8 1, label [[SW2:%.*]] ; CHECK-NEXT: ] ; CHECK: sw1: ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]] @@ -1040,10 +1039,9 @@ epilog: define i1 @PR24766_no_constants(i8 %x1, i8 %x2, i8 %condition, i1 %another_condition) { ; CHECK-LABEL: @PR24766_no_constants( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32 -; CHECK-NEXT: switch i32 [[CONV]], label [[EPILOG:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW1:%.*]] -; CHECK-NEXT: i32 1, label [[SW2:%.*]] +; CHECK-NEXT: switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [ +; CHECK-NEXT: i8 0, label [[SW1:%.*]] +; CHECK-NEXT: i8 1, label [[SW2:%.*]] ; CHECK-NEXT: ] ; CHECK: sw1: ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]] @@ -1085,10 +1083,9 @@ epilog: define i1 @PR24766_two_constants(i8 %x1, i8 %x2, i8 %condition) { ; CHECK-LABEL: @PR24766_two_constants( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32 -; CHECK-NEXT: switch i32 [[CONV]], label [[EPILOG:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW1:%.*]] -; CHECK-NEXT: i32 1, label [[SW2:%.*]] +; CHECK-NEXT: switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [ +; CHECK-NEXT: i8 0, label [[SW1:%.*]] +; CHECK-NEXT: i8 1, label [[SW2:%.*]] ; CHECK-NEXT: ] ; CHECK: sw1: ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]] @@ -1128,11 +1125,10 @@ epilog: define i1 @PR24766_two_constants_two_var(i8 %x1, i8 %x2, i8 %condition) { ; CHECK-LABEL: @PR24766_two_constants_two_var( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[CONDITION:%.*]] to i32 -; CHECK-NEXT: switch i32 [[CONV]], label [[EPILOG:%.*]] [ -; CHECK-NEXT: i32 0, label [[SW1:%.*]] -; CHECK-NEXT: i32 1, label [[SW2:%.*]] -; CHECK-NEXT: i32 2, label [[SW3:%.*]] +; CHECK-NEXT: switch i8 [[CONDITION:%.*]], label [[EPILOG:%.*]] [ +; CHECK-NEXT: i8 0, label [[SW1:%.*]] +; CHECK-NEXT: i8 1, label [[SW2:%.*]] +; CHECK-NEXT: i8 2, label [[SW3:%.*]] ; CHECK-NEXT: ] ; CHECK: sw1: ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[X1:%.*]], [[X2:%.*]] diff --git a/llvm/test/Transforms/InstCombine/switch-zext-sext.ll b/llvm/test/Transforms/InstCombine/switch-zext-sext.ll new file mode 100644 index 000000000000..c09441352acc --- /dev/null +++ b/llvm/test/Transforms/InstCombine/switch-zext-sext.ll @@ -0,0 +1,122 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt %s -passes=instcombine -S | FileCheck %s + +define i1 @test_switch_with_zext(i16 %a, i1 %b, i1 %c) { +; CHECK-LABEL: define i1 @test_switch_with_zext( +; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: switch i16 [[A]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i16 37, label [[SW_BB:%.*]] +; CHECK-NEXT: i16 38, label [[SW_BB]] +; CHECK-NEXT: i16 39, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 [[B]] +; CHECK: sw.default: +; CHECK-NEXT: ret i1 [[C]] +; +entry: + %a.ext = zext i16 %a to i32 + switch i32 %a.ext, label %sw.default [ + i32 37, label %sw.bb + i32 38, label %sw.bb + i32 39, label %sw.bb + ] + +sw.bb: + ret i1 %b +sw.default: + ret i1 %c +} + +define i1 @test_switch_with_sext(i16 %a, i1 %b, i1 %c) { +; CHECK-LABEL: define i1 @test_switch_with_sext( +; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: switch i16 [[A]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i16 37, label [[SW_BB:%.*]] +; CHECK-NEXT: i16 38, label [[SW_BB]] +; CHECK-NEXT: i16 39, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 [[B]] +; CHECK: sw.default: +; CHECK-NEXT: ret i1 [[C]] +; +entry: + %a.ext = sext i16 %a to i32 + switch i32 %a.ext, label %sw.default [ + i32 37, label %sw.bb + i32 38, label %sw.bb + i32 39, label %sw.bb + ] + +sw.bb: + ret i1 %b +sw.default: + ret i1 %c +} + +; Negative tests + +define i1 @test_switch_with_zext_unreachable_case(i16 %a, i1 %b, i1 %c) { +; CHECK-LABEL: define i1 @test_switch_with_zext_unreachable_case( +; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_EXT:%.*]] = zext i16 [[A]] to i32 +; CHECK-NEXT: switch i32 [[A_EXT]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 37, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 38, label [[SW_BB]] +; CHECK-NEXT: i32 39, label [[SW_BB]] +; CHECK-NEXT: i32 65537, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 [[B]] +; CHECK: sw.default: +; CHECK-NEXT: ret i1 [[C]] +; +entry: + %a.ext = zext i16 %a to i32 + switch i32 %a.ext, label %sw.default [ + i32 37, label %sw.bb + i32 38, label %sw.bb + i32 39, label %sw.bb + i32 65537, label %sw.bb + ] + +sw.bb: + ret i1 %b +sw.default: + ret i1 %c +} + +define i1 @test_switch_with_sext_unreachable_case(i16 %a, i1 %b, i1 %c) { +; CHECK-LABEL: define i1 @test_switch_with_sext_unreachable_case( +; CHECK-SAME: i16 [[A:%.*]], i1 [[B:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A_EXT:%.*]] = sext i16 [[A]] to i32 +; CHECK-NEXT: switch i32 [[A_EXT]], label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 37, label [[SW_BB:%.*]] +; CHECK-NEXT: i32 38, label [[SW_BB]] +; CHECK-NEXT: i32 39, label [[SW_BB]] +; CHECK-NEXT: i32 -65537, label [[SW_BB]] +; CHECK-NEXT: ] +; CHECK: sw.bb: +; CHECK-NEXT: ret i1 [[B]] +; CHECK: sw.default: +; CHECK-NEXT: ret i1 [[C]] +; +entry: + %a.ext = sext i16 %a to i32 + switch i32 %a.ext, label %sw.default [ + i32 37, label %sw.bb + i32 38, label %sw.bb + i32 39, label %sw.bb + i32 -65537, label %sw.bb + ] + +sw.bb: + ret i1 %b +sw.default: + ret i1 %c +} diff --git a/llvm/test/Transforms/PhaseOrdering/switch-sext.ll b/llvm/test/Transforms/PhaseOrdering/switch-sext.ll new file mode 100644 index 000000000000..e39771b629e0 --- /dev/null +++ b/llvm/test/Transforms/PhaseOrdering/switch-sext.ll @@ -0,0 +1,48 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes='default' < %s | FileCheck %s + +define i8 @test_switch_with_sext_phi(i8 %code) { +; CHECK-LABEL: define noundef i8 @test_switch_with_sext_phi( +; CHECK-SAME: i8 [[CODE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: switch i8 [[CODE]], label [[SW_EPILOG:%.*]] [ +; CHECK-NEXT: i8 76, label [[SW_BB3:%.*]] +; CHECK-NEXT: i8 108, label [[SW_BB2:%.*]] +; CHECK-NEXT: ] +; CHECK: sw.bb2: +; CHECK-NEXT: br label [[SW_EPILOG]] +; CHECK: sw.bb3: +; CHECK-NEXT: br label [[SW_EPILOG]] +; CHECK: sw.epilog: +; CHECK-NEXT: [[PEP_CODE:%.*]] = phi i8 [ 81, [[SW_BB3]] ], [ 113, [[SW_BB2]] ], [ [[CODE]], [[ENTRY:%.*]] ] +; CHECK-NEXT: ret i8 [[PEP_CODE]] +; +entry: + %conv = sext i8 %code to i32 + switch i32 %conv, label %sw.default [ + i32 105, label %sw.epilog + i32 73, label %sw.bb1 + i32 108, label %sw.bb2 + i32 76, label %sw.bb3 + i32 63, label %sw.bb4 + ] + +sw.bb1: + br label %sw.epilog + +sw.bb2: + br label %sw.epilog + +sw.bb3: + br label %sw.epilog + +sw.bb4: + br label %sw.epilog + +sw.default: + br label %sw.epilog + +sw.epilog: + %pep_code = phi i8 [ %code, %sw.default ], [ 63, %sw.bb4 ], [ 81, %sw.bb3 ], [ 113, %sw.bb2 ], [ 73, %sw.bb1 ], [ 105, %entry ] + ret i8 %pep_code +} -- GitLab From fc1c478709e380164733560e4a2c8f9e8d5e2c1c Mon Sep 17 00:00:00 2001 From: Jordan R AW <103465530+ajordanr-google@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:56:02 -0800 Subject: [PATCH 150/728] [libunwind] Replace process_vm_readv with SYS_rt_sigprocmask (#74791) process_vm_readv is generally considered dangerous from a syscall perspective, and is frequently blanket banned in seccomp filters such as those in Chromium and ChromiumOS. We can get the same behaviour during the invalid PC address case with the raw SYS_rt_sigprocmask syscall. Testing to ensure that process_vm_readv does not appear, I ran the output of check-unwind on an ARM64 device under strace. Previously, bad_unwind_info in particular would use process_vm_readv, but with this commit, it now no longer uses it: ``` strace test/Output/bad_unwind_info.pass.cpp.dir/t.tmp.exe \ |& grep process_vm_readv ``` The libunwind unittests were also tested on ARM64 ChromeOS (Gentoo Linux) devices. --- libunwind/src/UnwindCursor.hpp | 77 ++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp index 647a5a9c9d92..8517d328bd05 100644 --- a/libunwind/src/UnwindCursor.hpp +++ b/libunwind/src/UnwindCursor.hpp @@ -12,6 +12,8 @@ #define __UNWINDCURSOR_HPP__ #include "cet_unwind.h" +#include +#include #include #include #include @@ -990,6 +992,7 @@ private: R dummy; return stepThroughSigReturn(dummy); } + bool isReadableAddr(const pint_t addr) const; #if defined(_LIBUNWIND_TARGET_AARCH64) bool setInfoForSigReturn(Registers_arm64 &); int stepThroughSigReturn(Registers_arm64 &); @@ -2700,20 +2703,12 @@ bool UnwindCursor::setInfoForSigReturn(Registers_arm64 &) { // [1] https://github.com/torvalds/linux/blob/master/arch/arm64/kernel/vdso/sigreturn.S const pint_t pc = static_cast(this->getReg(UNW_REG_IP)); // The PC might contain an invalid address if the unwind info is bad, so - // directly accessing it could cause a segfault. Use process_vm_readv to read - // the memory safely instead. process_vm_readv was added in Linux 3.2, and - // AArch64 supported was added in Linux 3.7, so the syscall is guaranteed to - // be present. Unfortunately, there are Linux AArch64 environments where the - // libc wrapper for the syscall might not be present (e.g. Android 5), so call - // the syscall directly instead. - uint32_t instructions[2]; - struct iovec local_iov = {&instructions, sizeof instructions}; - struct iovec remote_iov = {reinterpret_cast(pc), sizeof instructions}; - long bytesRead = - syscall(SYS_process_vm_readv, getpid(), &local_iov, 1, &remote_iov, 1, 0); + // directly accessing it could cause a SIGSEGV. + if (!isReadableAddr(pc)) + return false; + auto *instructions = reinterpret_cast(pc); // Look for instructions: mov x8, #0x8b; svc #0x0 - if (bytesRead != sizeof instructions || instructions[0] != 0xd2801168 || - instructions[1] != 0xd4000001) + if (instructions[0] != 0xd2801168 || instructions[1] != 0xd4000001) return false; _info = {}; @@ -2762,18 +2757,17 @@ int UnwindCursor::stepThroughSigReturn(Registers_arm64 &) { template bool UnwindCursor::setInfoForSigReturn(Registers_riscv &) { const pint_t pc = static_cast(getReg(UNW_REG_IP)); - uint32_t instructions[2]; - struct iovec local_iov = {&instructions, sizeof instructions}; - struct iovec remote_iov = {reinterpret_cast(pc), sizeof instructions}; - long bytesRead = - syscall(SYS_process_vm_readv, getpid(), &local_iov, 1, &remote_iov, 1, 0); + // The PC might contain an invalid address if the unwind info is bad, so + // directly accessing it could cause a SIGSEGV. + if (!isReadableAddr(pc)) + return false; + const auto *instructions = reinterpret_cast(pc); // Look for the two instructions used in the sigreturn trampoline // __vdso_rt_sigreturn: // // 0x08b00893 li a7,0x8b // 0x00000073 ecall - if (bytesRead != sizeof instructions || instructions[0] != 0x08b00893 || - instructions[1] != 0x00000073) + if (instructions[0] != 0x08b00893 || instructions[1] != 0x00000073) return false; _info = {}; @@ -2822,13 +2816,11 @@ bool UnwindCursor::setInfoForSigReturn(Registers_s390x &) { // onto the stack. const pint_t pc = static_cast(this->getReg(UNW_REG_IP)); // The PC might contain an invalid address if the unwind info is bad, so - // directly accessing it could cause a segfault. Use process_vm_readv to - // read the memory safely instead. - uint16_t inst; - struct iovec local_iov = {&inst, sizeof inst}; - struct iovec remote_iov = {reinterpret_cast(pc), sizeof inst}; - long bytesRead = process_vm_readv(getpid(), &local_iov, 1, &remote_iov, 1, 0); - if (bytesRead == sizeof inst && (inst == 0x0a77 || inst == 0x0aad)) { + // directly accessing it could cause a SIGSEGV. + if (!isReadableAddr(pc)) + return false; + const auto inst = *reinterpret_cast(pc); + if (inst == 0x0a77 || inst == 0x0aad) { _info = {}; _info.start_ip = pc; _info.end_ip = pc + 2; @@ -2974,6 +2966,37 @@ bool UnwindCursor::getFunctionName(char *buf, size_t bufLen, buf, bufLen, offset); } +#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) +template +bool UnwindCursor::isReadableAddr(const pint_t addr) const { + // We use SYS_rt_sigprocmask, inspired by Abseil's AddressIsReadable. + + const auto sigsetAddr = reinterpret_cast(addr); + // We have to check that addr is nullptr because sigprocmask allows that + // as an argument without failure. + if (!sigsetAddr) + return false; + const auto saveErrno = errno; + // We MUST use a raw syscall here, as wrappers may try to access + // sigsetAddr which may cause a SIGSEGV. A raw syscall however is + // safe. Additionally, we need to pass the kernel_sigset_size, which is + // different from libc sizeof(sigset_t). For the majority of architectures, + // it's 64 bits (_NSIG), and libc NSIG is _NSIG + 1. + const auto kernelSigsetSize = NSIG / 8; + [[maybe_unused]] const int Result = syscall( + SYS_rt_sigprocmask, /*how=*/~0, sigsetAddr, nullptr, kernelSigsetSize); + // Because our "how" is invalid, this syscall should always fail, and our + // errno should always be EINVAL or an EFAULT. This relies on the Linux + // kernel to check copy_from_user before checking if the "how" argument is + // invalid. + assert(Result == -1); + assert(errno == EFAULT || errno == EINVAL); + const auto readable = errno != EFAULT; + errno = saveErrno; + return readable; +} +#endif + #if defined(_LIBUNWIND_USE_CET) extern "C" void *__libunwind_cet_get_registers(unw_cursor_t *cursor) { AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; -- GitLab From 365fbbfbcfefb8766f7716109b9c3767b58e6058 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 5 Jan 2024 21:04:01 +0000 Subject: [PATCH 151/728] [AArch64] Add some extra tests for SelectOpt. NFC --- llvm/test/CodeGen/AArch64/selectopt.ll | 593 +++++++++++++++++++++++++ 1 file changed, 593 insertions(+) diff --git a/llvm/test/CodeGen/AArch64/selectopt.ll b/llvm/test/CodeGen/AArch64/selectopt.ll index 8922eba0406c..f2b86dcb9211 100644 --- a/llvm/test/CodeGen/AArch64/selectopt.ll +++ b/llvm/test/CodeGen/AArch64/selectopt.ll @@ -268,3 +268,596 @@ if.end87: ; preds = %if.then, %while.bod while.end: ; preds = %land.rhs, %if.end87, %entry ret void } + + +define void @replace_or(ptr nocapture noundef %newst, ptr noundef %t, ptr noundef %h, i64 noundef %c, i64 noundef %rc, i64 noundef %ma, i64 noundef %n) { +; CHECKOO-LABEL: @replace_or( +; CHECKOO-NEXT: entry: +; CHECKOO-NEXT: [[T1:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], ptr [[NEWST:%.*]], i64 0, i32 2 +; CHECKOO-NEXT: store ptr [[T:%.*]], ptr [[T1]], align 8 +; CHECKOO-NEXT: [[H3:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 3 +; CHECKOO-NEXT: store ptr [[H:%.*]], ptr [[H3]], align 8 +; CHECKOO-NEXT: [[ORG_C:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 8 +; CHECKOO-NEXT: store i64 [[C:%.*]], ptr [[ORG_C]], align 8 +; CHECKOO-NEXT: [[C6:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 1 +; CHECKOO-NEXT: store i64 [[C]], ptr [[C6]], align 8 +; CHECKOO-NEXT: [[FLOW:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 7 +; CHECKOO-NEXT: store i64 [[RC:%.*]], ptr [[FLOW]], align 8 +; CHECKOO-NEXT: [[CONV:%.*]] = trunc i64 [[N:%.*]] to i32 +; CHECKOO-NEXT: store i32 [[CONV]], ptr [[NEWST]], align 8 +; CHECKOO-NEXT: [[FLOW10:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 1, i32 7 +; CHECKOO-NEXT: [[TMP0:%.*]] = load i64, ptr [[FLOW10]], align 8 +; CHECKOO-NEXT: [[FLOW12:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 2, i32 7 +; CHECKOO-NEXT: [[TMP1:%.*]] = load i64, ptr [[FLOW12]], align 8 +; CHECKOO-NEXT: [[CMP13:%.*]] = icmp sgt i64 [[TMP0]], [[TMP1]] +; CHECKOO-NEXT: [[CONV15:%.*]] = select i1 [[CMP13]], i64 2, i64 3 +; CHECKOO-NEXT: [[CMP16_NOT149:%.*]] = icmp sgt i64 [[CONV15]], [[MA:%.*]] +; CHECKOO-NEXT: br i1 [[CMP16_NOT149]], label [[WHILE_END:%.*]], label [[LAND_RHS:%.*]] +; CHECKOO: land.rhs: +; CHECKOO-NEXT: [[CMP_0151:%.*]] = phi i64 [ [[CMP_1:%.*]], [[IF_END87:%.*]] ], [ [[CONV15]], [[ENTRY:%.*]] ] +; CHECKOO-NEXT: [[POS_0150:%.*]] = phi i64 [ [[CMP_0151]], [[IF_END87]] ], [ 1, [[ENTRY]] ] +; CHECKOO-NEXT: [[SUB:%.*]] = add nsw i64 [[CMP_0151]], -1 +; CHECKOO-NEXT: [[FLOW19:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 7 +; CHECKOO-NEXT: [[TMP2:%.*]] = load i64, ptr [[FLOW19]], align 8 +; CHECKOO-NEXT: [[CMP20:%.*]] = icmp sgt i64 [[TMP2]], [[RC]] +; CHECKOO-NEXT: br i1 [[CMP20]], label [[WHILE_BODY:%.*]], label [[WHILE_END]] +; CHECKOO: while.body: +; CHECKOO-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]] +; CHECKOO-NEXT: [[T24:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 2 +; CHECKOO-NEXT: [[TMP3:%.*]] = load ptr, ptr [[T24]], align 8 +; CHECKOO-NEXT: [[SUB25:%.*]] = add nsw i64 [[POS_0150]], -1 +; CHECKOO-NEXT: [[ARRAYIDX26:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]] +; CHECKOO-NEXT: [[T27:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 2 +; CHECKOO-NEXT: store ptr [[TMP3]], ptr [[T27]], align 8 +; CHECKOO-NEXT: [[H30:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 3 +; CHECKOO-NEXT: [[TMP4:%.*]] = load ptr, ptr [[H30]], align 8 +; CHECKOO-NEXT: [[H33:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 3 +; CHECKOO-NEXT: store ptr [[TMP4]], ptr [[H33]], align 8 +; CHECKOO-NEXT: [[C36:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 1 +; CHECKOO-NEXT: [[TMP5:%.*]] = load i64, ptr [[C36]], align 8 +; CHECKOO-NEXT: [[C39:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 1 +; CHECKOO-NEXT: store i64 [[TMP5]], ptr [[C39]], align 8 +; CHECKOO-NEXT: [[ORG_C45:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 8 +; CHECKOO-NEXT: store i64 [[TMP5]], ptr [[ORG_C45]], align 8 +; CHECKOO-NEXT: [[FLOW51:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 7 +; CHECKOO-NEXT: store i64 [[TMP2]], ptr [[FLOW51]], align 8 +; CHECKOO-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX18]], align 8 +; CHECKOO-NEXT: store i32 [[TMP6]], ptr [[ARRAYIDX26]], align 8 +; CHECKOO-NEXT: store ptr [[T]], ptr [[T24]], align 8 +; CHECKOO-NEXT: store ptr [[H]], ptr [[H30]], align 8 +; CHECKOO-NEXT: store i64 [[C]], ptr [[C36]], align 8 +; CHECKOO-NEXT: [[ORG_C69:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 8 +; CHECKOO-NEXT: store i64 [[C]], ptr [[ORG_C69]], align 8 +; CHECKOO-NEXT: store i64 [[RC]], ptr [[FLOW19]], align 8 +; CHECKOO-NEXT: store i32 [[CONV]], ptr [[ARRAYIDX18]], align 8 +; CHECKOO-NEXT: [[MUL:%.*]] = shl nsw i64 [[CMP_0151]], 1 +; CHECKOO-NEXT: [[CMP77_NOT_NOT:%.*]] = icmp slt i64 [[MUL]], [[MA]] +; CHECKOO-NEXT: br i1 [[CMP77_NOT_NOT]], label [[IF_THEN:%.*]], label [[IF_END87]] +; CHECKOO: if.then: +; CHECKOO-NEXT: [[SUB79:%.*]] = add nsw i64 [[MUL]], -1 +; CHECKOO-NEXT: [[FLOW81:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB79]], i32 7 +; CHECKOO-NEXT: [[TMP7:%.*]] = load i64, ptr [[FLOW81]], align 8 +; CHECKOO-NEXT: [[FLOW83:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[MUL]], i32 7 +; CHECKOO-NEXT: [[TMP8:%.*]] = load i64, ptr [[FLOW83]], align 8 +; CHECKOO-NEXT: [[CMP84:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +; CHECKOO-NEXT: [[ADD:%.*]] = zext i1 [[CMP84]] to i64 +; CHECKOO-NEXT: [[SPEC_SELECT:%.*]] = or disjoint i64 [[MUL]], [[ADD]] +; CHECKOO-NEXT: br label [[IF_END87]] +; CHECKOO: if.end87: +; CHECKOO-NEXT: [[CMP_1]] = phi i64 [ [[MUL]], [[WHILE_BODY]] ], [ [[SPEC_SELECT]], [[IF_THEN]] ] +; CHECKOO-NEXT: [[CMP16_NOT:%.*]] = icmp sgt i64 [[CMP_1]], [[MA]] +; CHECKOO-NEXT: br i1 [[CMP16_NOT]], label [[WHILE_END]], label [[LAND_RHS]] +; CHECKOO: while.end: +; CHECKOO-NEXT: ret void +; +; CHECKII-LABEL: @replace_or( +; CHECKII-NEXT: entry: +; CHECKII-NEXT: [[T1:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], ptr [[NEWST:%.*]], i64 0, i32 2 +; CHECKII-NEXT: store ptr [[T:%.*]], ptr [[T1]], align 8 +; CHECKII-NEXT: [[H3:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 3 +; CHECKII-NEXT: store ptr [[H:%.*]], ptr [[H3]], align 8 +; CHECKII-NEXT: [[ORG_C:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 8 +; CHECKII-NEXT: store i64 [[C:%.*]], ptr [[ORG_C]], align 8 +; CHECKII-NEXT: [[C6:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 1 +; CHECKII-NEXT: store i64 [[C]], ptr [[C6]], align 8 +; CHECKII-NEXT: [[FLOW:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 0, i32 7 +; CHECKII-NEXT: store i64 [[RC:%.*]], ptr [[FLOW]], align 8 +; CHECKII-NEXT: [[CONV:%.*]] = trunc i64 [[N:%.*]] to i32 +; CHECKII-NEXT: store i32 [[CONV]], ptr [[NEWST]], align 8 +; CHECKII-NEXT: [[FLOW10:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 1, i32 7 +; CHECKII-NEXT: [[TMP0:%.*]] = load i64, ptr [[FLOW10]], align 8 +; CHECKII-NEXT: [[FLOW12:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 2, i32 7 +; CHECKII-NEXT: [[TMP1:%.*]] = load i64, ptr [[FLOW12]], align 8 +; CHECKII-NEXT: [[CMP13:%.*]] = icmp sgt i64 [[TMP0]], [[TMP1]] +; CHECKII-NEXT: [[CONV15:%.*]] = select i1 [[CMP13]], i64 2, i64 3 +; CHECKII-NEXT: [[CMP16_NOT149:%.*]] = icmp sgt i64 [[CONV15]], [[MA:%.*]] +; CHECKII-NEXT: br i1 [[CMP16_NOT149]], label [[WHILE_END:%.*]], label [[LAND_RHS:%.*]] +; CHECKII: land.rhs: +; CHECKII-NEXT: [[CMP_0151:%.*]] = phi i64 [ [[CMP_1:%.*]], [[IF_END87:%.*]] ], [ [[CONV15]], [[ENTRY:%.*]] ] +; CHECKII-NEXT: [[POS_0150:%.*]] = phi i64 [ [[CMP_0151]], [[IF_END87]] ], [ 1, [[ENTRY]] ] +; CHECKII-NEXT: [[SUB:%.*]] = add nsw i64 [[CMP_0151]], -1 +; CHECKII-NEXT: [[FLOW19:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 7 +; CHECKII-NEXT: [[TMP2:%.*]] = load i64, ptr [[FLOW19]], align 8 +; CHECKII-NEXT: [[CMP20:%.*]] = icmp sgt i64 [[TMP2]], [[RC]] +; CHECKII-NEXT: br i1 [[CMP20]], label [[WHILE_BODY:%.*]], label [[WHILE_END]] +; CHECKII: while.body: +; CHECKII-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]] +; CHECKII-NEXT: [[T24:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 2 +; CHECKII-NEXT: [[TMP3:%.*]] = load ptr, ptr [[T24]], align 8 +; CHECKII-NEXT: [[SUB25:%.*]] = add nsw i64 [[POS_0150]], -1 +; CHECKII-NEXT: [[ARRAYIDX26:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]] +; CHECKII-NEXT: [[T27:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 2 +; CHECKII-NEXT: store ptr [[TMP3]], ptr [[T27]], align 8 +; CHECKII-NEXT: [[H30:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 3 +; CHECKII-NEXT: [[TMP4:%.*]] = load ptr, ptr [[H30]], align 8 +; CHECKII-NEXT: [[H33:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 3 +; CHECKII-NEXT: store ptr [[TMP4]], ptr [[H33]], align 8 +; CHECKII-NEXT: [[C36:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 1 +; CHECKII-NEXT: [[TMP5:%.*]] = load i64, ptr [[C36]], align 8 +; CHECKII-NEXT: [[C39:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 1 +; CHECKII-NEXT: store i64 [[TMP5]], ptr [[C39]], align 8 +; CHECKII-NEXT: [[ORG_C45:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 8 +; CHECKII-NEXT: store i64 [[TMP5]], ptr [[ORG_C45]], align 8 +; CHECKII-NEXT: [[FLOW51:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB25]], i32 7 +; CHECKII-NEXT: store i64 [[TMP2]], ptr [[FLOW51]], align 8 +; CHECKII-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX18]], align 8 +; CHECKII-NEXT: store i32 [[TMP6]], ptr [[ARRAYIDX26]], align 8 +; CHECKII-NEXT: store ptr [[T]], ptr [[T24]], align 8 +; CHECKII-NEXT: store ptr [[H]], ptr [[H30]], align 8 +; CHECKII-NEXT: store i64 [[C]], ptr [[C36]], align 8 +; CHECKII-NEXT: [[ORG_C69:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB]], i32 8 +; CHECKII-NEXT: store i64 [[C]], ptr [[ORG_C69]], align 8 +; CHECKII-NEXT: store i64 [[RC]], ptr [[FLOW19]], align 8 +; CHECKII-NEXT: store i32 [[CONV]], ptr [[ARRAYIDX18]], align 8 +; CHECKII-NEXT: [[MUL:%.*]] = shl nsw i64 [[CMP_0151]], 1 +; CHECKII-NEXT: [[CMP77_NOT_NOT:%.*]] = icmp slt i64 [[MUL]], [[MA]] +; CHECKII-NEXT: br i1 [[CMP77_NOT_NOT]], label [[IF_THEN:%.*]], label [[IF_END87]] +; CHECKII: if.then: +; CHECKII-NEXT: [[SUB79:%.*]] = add nsw i64 [[MUL]], -1 +; CHECKII-NEXT: [[FLOW81:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[SUB79]], i32 7 +; CHECKII-NEXT: [[TMP7:%.*]] = load i64, ptr [[FLOW81]], align 8 +; CHECKII-NEXT: [[FLOW83:%.*]] = getelementptr inbounds [[STRUCT_ST]], ptr [[NEWST]], i64 [[MUL]], i32 7 +; CHECKII-NEXT: [[TMP8:%.*]] = load i64, ptr [[FLOW83]], align 8 +; CHECKII-NEXT: [[CMP84:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +; CHECKII-NEXT: [[ADD:%.*]] = zext i1 [[CMP84]] to i64 +; CHECKII-NEXT: [[SPEC_SELECT:%.*]] = or disjoint i64 [[MUL]], [[ADD]] +; CHECKII-NEXT: br label [[IF_END87]] +; CHECKII: if.end87: +; CHECKII-NEXT: [[CMP_1]] = phi i64 [ [[MUL]], [[WHILE_BODY]] ], [ [[SPEC_SELECT]], [[IF_THEN]] ] +; CHECKII-NEXT: [[CMP16_NOT:%.*]] = icmp sgt i64 [[CMP_1]], [[MA]] +; CHECKII-NEXT: br i1 [[CMP16_NOT]], label [[WHILE_END]], label [[LAND_RHS]] +; CHECKII: while.end: +; CHECKII-NEXT: ret void +; +entry: + %t1 = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 2 + store ptr %t, ptr %t1, align 8 + %h3 = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 3 + store ptr %h, ptr %h3, align 8 + %org_c = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 8 + store i64 %c, ptr %org_c, align 8 + %c6 = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 1 + store i64 %c, ptr %c6, align 8 + %flow = getelementptr inbounds %struct.st, ptr %newst, i64 0, i32 7 + store i64 %rc, ptr %flow, align 8 + %conv = trunc i64 %n to i32 + store i32 %conv, ptr %newst, align 8 + %flow10 = getelementptr inbounds %struct.st, ptr %newst, i64 1, i32 7 + %0 = load i64, ptr %flow10, align 8 + %flow12 = getelementptr inbounds %struct.st, ptr %newst, i64 2, i32 7 + %1 = load i64, ptr %flow12, align 8 + %cmp13 = icmp sgt i64 %0, %1 + %conv15 = select i1 %cmp13, i64 2, i64 3 + %cmp16.not149 = icmp sgt i64 %conv15, %ma + br i1 %cmp16.not149, label %while.end, label %land.rhs + +land.rhs: ; preds = %entry, %if.end87 + %cmp.0151 = phi i64 [ %cmp.1, %if.end87 ], [ %conv15, %entry ] + %pos.0150 = phi i64 [ %cmp.0151, %if.end87 ], [ 1, %entry ] + %sub = add nsw i64 %cmp.0151, -1 + %flow19 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 7 + %2 = load i64, ptr %flow19, align 8 + %cmp20 = icmp sgt i64 %2, %rc + br i1 %cmp20, label %while.body, label %while.end + +while.body: ; preds = %land.rhs + %arrayidx18 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub + %t24 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 2 + %3 = load ptr, ptr %t24, align 8 + %sub25 = add nsw i64 %pos.0150, -1 + %arrayidx26 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25 + %t27 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 2 + store ptr %3, ptr %t27, align 8 + %h30 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 3 + %4 = load ptr, ptr %h30, align 8 + %h33 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 3 + store ptr %4, ptr %h33, align 8 + %c36 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 1 + %5 = load i64, ptr %c36, align 8 + %c39 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 1 + store i64 %5, ptr %c39, align 8 + %org_c45 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 8 + store i64 %5, ptr %org_c45, align 8 + %flow51 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub25, i32 7 + store i64 %2, ptr %flow51, align 8 + %6 = load i32, ptr %arrayidx18, align 8 + store i32 %6, ptr %arrayidx26, align 8 + store ptr %t, ptr %t24, align 8 + store ptr %h, ptr %h30, align 8 + store i64 %c, ptr %c36, align 8 + %org_c69 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub, i32 8 + store i64 %c, ptr %org_c69, align 8 + store i64 %rc, ptr %flow19, align 8 + store i32 %conv, ptr %arrayidx18, align 8 + %mul = shl nsw i64 %cmp.0151, 1 + %cmp77.not.not = icmp slt i64 %mul, %ma + br i1 %cmp77.not.not, label %if.then, label %if.end87 + +if.then: ; preds = %while.body + %sub79 = add nsw i64 %mul, -1 + %flow81 = getelementptr inbounds %struct.st, ptr %newst, i64 %sub79, i32 7 + %7 = load i64, ptr %flow81, align 8 + %flow83 = getelementptr inbounds %struct.st, ptr %newst, i64 %mul, i32 7 + %8 = load i64, ptr %flow83, align 8 + %cmp84 = icmp slt i64 %7, %8 + %add = zext i1 %cmp84 to i64 + %spec.select = or disjoint i64 %mul, %add + br label %if.end87 + +if.end87: ; preds = %if.then, %while.body + %cmp.1 = phi i64 [ %mul, %while.body ], [ %spec.select, %if.then ] + %cmp16.not = icmp sgt i64 %cmp.1, %ma + br i1 %cmp16.not, label %while.end, label %land.rhs + +while.end: ; preds = %if.end87, %land.rhs, %entry + ret void +} + + +; This `or` is not transformed as it is not the last instruction in the block +define i32 @or_notatendofblock(ptr nocapture noundef %x, i32 noundef %n, ptr nocapture noundef readonly %z) { +; CHECKOO-LABEL: @or_notatendofblock( +; CHECKOO-NEXT: entry: +; CHECKOO-NEXT: [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0 +; CHECKOO-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECKOO: for.body.preheader: +; CHECKOO-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 +; CHECKOO-NEXT: br label [[FOR_BODY:%.*]] +; CHECKOO: for.cond.cleanup: +; CHECKOO-NEXT: [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ] +; CHECKOO-NEXT: ret i32 [[Y_0_LCSSA]] +; CHECKOO: for.body: +; CHECKOO-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ] +; CHECKOO-NEXT: [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ] +; CHECKOO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]] +; CHECKOO-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECKOO-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]] +; CHECKOO-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0 +; CHECKOO-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECKOO: if.then: +; CHECKOO-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]] +; CHECKOO-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4 +; CHECKOO-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP0]], [[TMP1]] +; CHECKOO-NEXT: [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]] +; CHECKOO-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV1]], [[TMP1]] +; CHECKOO-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[DIV2]], 0 +; CHECKOO-NEXT: [[CONV:%.*]] = zext i1 [[CMP5]] to i32 +; CHECKOO-NEXT: [[OR1:%.*]] = or i32 [[CONV]], [[ADD]] +; CHECKOO-NEXT: [[OR:%.*]] = add i32 [[OR1]], 1 +; CHECKOO-NEXT: br label [[IF_END]] +; CHECKOO: if.end: +; CHECKOO-NEXT: [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ] +; CHECKOO-NEXT: store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4 +; CHECKOO-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; CHECKOO-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]] +; CHECKOO-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]] +; +; CHECKII-LABEL: @or_notatendofblock( +; CHECKII-NEXT: entry: +; CHECKII-NEXT: [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0 +; CHECKII-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECKII: for.body.preheader: +; CHECKII-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 +; CHECKII-NEXT: br label [[FOR_BODY:%.*]] +; CHECKII: for.cond.cleanup: +; CHECKII-NEXT: [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ] +; CHECKII-NEXT: ret i32 [[Y_0_LCSSA]] +; CHECKII: for.body: +; CHECKII-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ] +; CHECKII-NEXT: [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ] +; CHECKII-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]] +; CHECKII-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECKII-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]] +; CHECKII-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0 +; CHECKII-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECKII: if.then: +; CHECKII-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]] +; CHECKII-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4 +; CHECKII-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP0]], [[TMP1]] +; CHECKII-NEXT: [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]] +; CHECKII-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV1]], [[TMP1]] +; CHECKII-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[DIV2]], 0 +; CHECKII-NEXT: [[CONV:%.*]] = zext i1 [[CMP5]] to i32 +; CHECKII-NEXT: [[OR1:%.*]] = or i32 [[CONV]], [[ADD]] +; CHECKII-NEXT: [[OR:%.*]] = add i32 [[OR1]], 1 +; CHECKII-NEXT: br label [[IF_END]] +; CHECKII: if.end: +; CHECKII-NEXT: [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ] +; CHECKII-NEXT: store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4 +; CHECKII-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; CHECKII-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]] +; CHECKII-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]] +; +entry: + %cmp19 = icmp sgt i32 %n, 0 + br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup + +for.body.preheader: + %wide.trip.count = zext nneg i32 %n to i64 + br label %for.body + +for.cond.cleanup: + %y.0.lcssa = phi i32 [ 0, %entry ], [ %y.1, %if.end ] + ret i32 %y.0.lcssa + +for.body: + %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %if.end ] + %y.020 = phi i32 [ 0, %for.body.preheader ], [ %y.1, %if.end ] + %arrayidx = getelementptr inbounds i32, ptr %x, i64 %indvars.iv + %0 = load i32, ptr %arrayidx, align 4 + %add = add nsw i32 %0, %y.020 + %tobool.not = icmp eq i32 %add, 0 + br i1 %tobool.not, label %if.end, label %if.then + +if.then: + %arrayidx4 = getelementptr inbounds i32, ptr %z, i64 %indvars.iv + %1 = load i32, ptr %arrayidx4, align 4 + %div = sdiv i32 %0, %1 + %div1 = sdiv i32 %div, %1 + %div2 = sdiv i32 %div1, %1 + %cmp5 = icmp sgt i32 %div2, 0 + %conv = zext i1 %cmp5 to i32 + %or1 = or i32 %conv, %add + %or = add i32 %or1, 1 + br label %if.end + +if.end: + %y.1 = phi i32 [ %or, %if.then ], [ 0, %for.body ] + store i32 %y.1, ptr %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} + + +; Similar to the last test, an artificial test with the or as the last instruction and a select in the same group. +define i32 @or_samegroup(ptr nocapture noundef %x, i32 noundef %n, ptr nocapture noundef readonly %z) { +; CHECKOO-LABEL: @or_samegroup( +; CHECKOO-NEXT: entry: +; CHECKOO-NEXT: [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0 +; CHECKOO-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECKOO: for.body.preheader: +; CHECKOO-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 +; CHECKOO-NEXT: br label [[FOR_BODY:%.*]] +; CHECKOO: for.cond.cleanup: +; CHECKOO-NEXT: [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ] +; CHECKOO-NEXT: ret i32 [[Y_0_LCSSA]] +; CHECKOO: for.body: +; CHECKOO-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ] +; CHECKOO-NEXT: [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ] +; CHECKOO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]] +; CHECKOO-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECKOO-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]] +; CHECKOO-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0 +; CHECKOO-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECKOO: if.then: +; CHECKOO-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]] +; CHECKOO-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4 +; CHECKOO-NEXT: [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]] +; CHECKOO-NEXT: [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]] +; CHECKOO-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0 +; CHECKOO-NEXT: [[CONV:%.*]] = zext i1 [[CMP5]] to i32 +; CHECKOO-NEXT: [[SEL_FROZEN:%.*]] = freeze i1 [[CMP5]] +; CHECKOO-NEXT: br i1 [[SEL_FROZEN]], label [[SELECT_END:%.*]], label [[SELECT_FALSE:%.*]] +; CHECKOO: select.false: +; CHECKOO-NEXT: br label [[SELECT_END]] +; CHECKOO: select.end: +; CHECKOO-NEXT: [[SEL:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ 1, [[SELECT_FALSE]] ] +; CHECKOO-NEXT: [[OR:%.*]] = or i32 [[CONV]], [[SEL]] +; CHECKOO-NEXT: br label [[IF_END]] +; CHECKOO: if.end: +; CHECKOO-NEXT: [[Y_1]] = phi i32 [ [[SEL]], [[SELECT_END]] ], [ 0, [[FOR_BODY]] ] +; CHECKOO-NEXT: store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4 +; CHECKOO-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; CHECKOO-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]] +; CHECKOO-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]] +; +; CHECKII-LABEL: @or_samegroup( +; CHECKII-NEXT: entry: +; CHECKII-NEXT: [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0 +; CHECKII-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECKII: for.body.preheader: +; CHECKII-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 +; CHECKII-NEXT: br label [[FOR_BODY:%.*]] +; CHECKII: for.cond.cleanup: +; CHECKII-NEXT: [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ] +; CHECKII-NEXT: ret i32 [[Y_0_LCSSA]] +; CHECKII: for.body: +; CHECKII-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ] +; CHECKII-NEXT: [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ] +; CHECKII-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]] +; CHECKII-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECKII-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]] +; CHECKII-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0 +; CHECKII-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECKII: if.then: +; CHECKII-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]] +; CHECKII-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4 +; CHECKII-NEXT: [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]] +; CHECKII-NEXT: [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]] +; CHECKII-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0 +; CHECKII-NEXT: [[CONV:%.*]] = zext i1 [[CMP5]] to i32 +; CHECKII-NEXT: [[SEL:%.*]] = select i1 [[CMP5]], i32 [[ADD]], i32 1 +; CHECKII-NEXT: [[OR:%.*]] = or i32 [[CONV]], [[SEL]] +; CHECKII-NEXT: br label [[IF_END]] +; CHECKII: if.end: +; CHECKII-NEXT: [[Y_1]] = phi i32 [ [[SEL]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ] +; CHECKII-NEXT: store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4 +; CHECKII-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; CHECKII-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]] +; CHECKII-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]] +; +entry: + %cmp19 = icmp sgt i32 %n, 0 + br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup + +for.body.preheader: + %wide.trip.count = zext nneg i32 %n to i64 + br label %for.body + +for.cond.cleanup: + %y.0.lcssa = phi i32 [ 0, %entry ], [ %y.1, %if.end ] + ret i32 %y.0.lcssa + +for.body: + %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %if.end ] + %y.020 = phi i32 [ 0, %for.body.preheader ], [ %y.1, %if.end ] + %arrayidx = getelementptr inbounds i32, ptr %x, i64 %indvars.iv + %0 = load i32, ptr %arrayidx, align 4 + %add = add nsw i32 %0, %y.020 + %tobool.not = icmp eq i32 %add, 0 + br i1 %tobool.not, label %if.end, label %if.then + +if.then: + %arrayidx4 = getelementptr inbounds i32, ptr %z, i64 %indvars.iv + %1 = load i32, ptr %arrayidx4, align 4 + %div = sdiv i32 %y.020, %1 + %div1 = sdiv i32 %div, %1 + %cmp5 = icmp sgt i32 %div1, 0 + %conv = zext i1 %cmp5 to i32 + %sel = select i1 %cmp5, i32 %add, i32 1 + %or = or i32 %conv, %sel + br label %if.end + +if.end: + %y.1 = phi i32 [ %sel, %if.then ], [ 0, %for.body ] + store i32 %y.1, ptr %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} + +; Same test again with a one use value group of values on the or +define i32 @or_oneusevalues(ptr nocapture noundef %x, i32 noundef %n, ptr nocapture noundef readonly %z) { +; CHECKOO-LABEL: @or_oneusevalues( +; CHECKOO-NEXT: entry: +; CHECKOO-NEXT: [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0 +; CHECKOO-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECKOO: for.body.preheader: +; CHECKOO-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 +; CHECKOO-NEXT: br label [[FOR_BODY:%.*]] +; CHECKOO: for.cond.cleanup: +; CHECKOO-NEXT: [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ] +; CHECKOO-NEXT: ret i32 [[Y_0_LCSSA]] +; CHECKOO: for.body: +; CHECKOO-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ] +; CHECKOO-NEXT: [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ] +; CHECKOO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]] +; CHECKOO-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECKOO-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]] +; CHECKOO-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0 +; CHECKOO-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECKOO: if.then: +; CHECKOO-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]] +; CHECKOO-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4 +; CHECKOO-NEXT: [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]] +; CHECKOO-NEXT: [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]] +; CHECKOO-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0 +; CHECKOO-NEXT: [[CONV:%.*]] = zext i1 [[CMP5]] to i32 +; CHECKOO-NEXT: [[ADD1:%.*]] = add i32 [[ADD]], 1 +; CHECKOO-NEXT: [[ADD2:%.*]] = or i32 [[ADD1]], 1 +; CHECKOO-NEXT: [[OR:%.*]] = or i32 [[CONV]], [[ADD2]] +; CHECKOO-NEXT: br label [[IF_END]] +; CHECKOO: if.end: +; CHECKOO-NEXT: [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ] +; CHECKOO-NEXT: store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4 +; CHECKOO-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; CHECKOO-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]] +; CHECKOO-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]] +; +; CHECKII-LABEL: @or_oneusevalues( +; CHECKII-NEXT: entry: +; CHECKII-NEXT: [[CMP19:%.*]] = icmp sgt i32 [[N:%.*]], 0 +; CHECKII-NEXT: br i1 [[CMP19]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]] +; CHECKII: for.body.preheader: +; CHECKII-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 +; CHECKII-NEXT: br label [[FOR_BODY:%.*]] +; CHECKII: for.cond.cleanup: +; CHECKII-NEXT: [[Y_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[Y_1:%.*]], [[IF_END:%.*]] ] +; CHECKII-NEXT: ret i32 [[Y_0_LCSSA]] +; CHECKII: for.body: +; CHECKII-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ] +; CHECKII-NEXT: [[Y_020:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[Y_1]], [[IF_END]] ] +; CHECKII-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 [[INDVARS_IV]] +; CHECKII-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECKII-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], [[Y_020]] +; CHECKII-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[ADD]], 0 +; CHECKII-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECKII: if.then: +; CHECKII-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 [[INDVARS_IV]] +; CHECKII-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4 +; CHECKII-NEXT: [[DIV:%.*]] = sdiv i32 [[Y_020]], [[TMP1]] +; CHECKII-NEXT: [[DIV1:%.*]] = sdiv i32 [[DIV]], [[TMP1]] +; CHECKII-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[DIV1]], 0 +; CHECKII-NEXT: [[CONV:%.*]] = zext i1 [[CMP5]] to i32 +; CHECKII-NEXT: [[ADD1:%.*]] = add i32 [[ADD]], 1 +; CHECKII-NEXT: [[ADD2:%.*]] = or i32 [[ADD1]], 1 +; CHECKII-NEXT: [[OR:%.*]] = or i32 [[CONV]], [[ADD2]] +; CHECKII-NEXT: br label [[IF_END]] +; CHECKII: if.end: +; CHECKII-NEXT: [[Y_1]] = phi i32 [ [[OR]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ] +; CHECKII-NEXT: store i32 [[Y_1]], ptr [[ARRAYIDX]], align 4 +; CHECKII-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 +; CHECKII-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]] +; CHECKII-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]] +; +entry: + %cmp19 = icmp sgt i32 %n, 0 + br i1 %cmp19, label %for.body.preheader, label %for.cond.cleanup + +for.body.preheader: + %wide.trip.count = zext nneg i32 %n to i64 + br label %for.body + +for.cond.cleanup: + %y.0.lcssa = phi i32 [ 0, %entry ], [ %y.1, %if.end ] + ret i32 %y.0.lcssa + +for.body: + %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %if.end ] + %y.020 = phi i32 [ 0, %for.body.preheader ], [ %y.1, %if.end ] + %arrayidx = getelementptr inbounds i32, ptr %x, i64 %indvars.iv + %0 = load i32, ptr %arrayidx, align 4 + %add = add nsw i32 %0, %y.020 + %tobool.not = icmp eq i32 %add, 0 + br i1 %tobool.not, label %if.end, label %if.then + +if.then: + %arrayidx4 = getelementptr inbounds i32, ptr %z, i64 %indvars.iv + %1 = load i32, ptr %arrayidx4, align 4 + %div = sdiv i32 %y.020, %1 + %div1 = sdiv i32 %div, %1 + %cmp5 = icmp sgt i32 %div1, 0 + %conv = zext i1 %cmp5 to i32 + %add1 = add i32 %add, 1 + %add2 = or i32 %add1, 1 + %or = or i32 %conv, %add2 + br label %if.end + +if.end: + %y.1 = phi i32 [ %or, %if.then ], [ 0, %for.body ] + store i32 %y.1, ptr %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count + br i1 %exitcond.not, label %for.cond.cleanup, label %for.body +} -- GitLab From dcdb4a36ae3549fb1852b2c0089247a8d477eaf3 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 5 Jan 2024 13:31:35 -0800 Subject: [PATCH 152/728] [libc][cmake] append per obj compile options instead of prepending (#77126) This allows individual object files to override the common compile commands in their local CMakeLists' add_object_library call. For example, the common compile commands contain -Wall and -Wextra. Before this patch, the per object COMPILE_OPTIONS were prepended to these, so that builds of individual object files could not individually disable specific diagnostics from those groups explicitly. After this patch, the per-object file compile objects are appended to the list of compiler flags, enabling this use case. ARGN is a bit of cmake magic; let's be explicit in the APPEND that we're appending the compile options. Link: #77007 --- libc/cmake/modules/LLVMLibCObjectRules.cmake | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/libc/cmake/modules/LLVMLibCObjectRules.cmake b/libc/cmake/modules/LLVMLibCObjectRules.cmake index 5fbbfd58db2d..c3e3fa2dccfe 100644 --- a/libc/cmake/modules/LLVMLibCObjectRules.cmake +++ b/libc/cmake/modules/LLVMLibCObjectRules.cmake @@ -26,7 +26,7 @@ function(_get_common_compile_options output_var flags) set(ADD_PREFER_GENERIC_FLAG TRUE) endif() - set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT} ${ARGN}) + set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT}) if(LLVM_COMPILER_IS_GCC_COMPATIBLE) list(APPEND compile_options "-fpie") @@ -357,11 +357,8 @@ function(create_object_library fq_target_name) set(internal_target_name ${fq_target_name}) endif() - _get_common_compile_options( - compile_options - "${ADD_OBJECT_FLAGS}" - ${ADD_OBJECT_COMPILE_OPTIONS} - ) + _get_common_compile_options(compile_options "${ADD_OBJECT_FLAGS}") + list(APPEND compile_options ${ADD_OBJECT_COMPILE_OPTIONS}) # GPU builds require special handling for the objects because we want to # export several different targets at once, e.g. for both Nvidia and AMD. @@ -640,11 +637,8 @@ function(create_entrypoint_object fq_target_name) set(ADD_ENTRYPOINT_OBJ_CXX_STANDARD ${CMAKE_CXX_STANDARD}) endif() - _get_common_compile_options( - common_compile_options - "${ADD_ENTRYPOINT_OBJ_FLAGS}" - ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS} - ) + _get_common_compile_options(common_compile_options "${ADD_ENTRYPOINT_OBJ_FLAGS}") + list(APPEND common_compile_options ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS}) get_fq_deps_list(fq_deps_list ${ADD_ENTRYPOINT_OBJ_DEPENDS}) set(full_deps_list ${fq_deps_list} libc.src.__support.common) -- GitLab From 6f4cc1310b12bc59210e4596a895db4cb9ad6075 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 5 Jan 2024 13:31:48 -0800 Subject: [PATCH 153/728] [llvm-exegesis] Remove unused Counter::read method (#76651) This method was simply a wrapper around readOrError. All users within the llvm-exegesis code base should have been processing an actual error rather than using the wrapper. This patch removes the wrapper and rewrites the users (just 1) to use the readOrError method. --- llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp | 6 +++++- llvm/tools/llvm-exegesis/lib/PerfHelper.cpp | 13 ------------- llvm/tools/llvm-exegesis/lib/PerfHelper.h | 3 --- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 5f08c67bfc89..dee7af5fd520 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -356,7 +356,11 @@ private: if (ChildExitCode == 0) { // The child exited succesfully, read counter values and return // success - CounterValues[0] = Counter->read(); + auto CounterValueOrErr = Counter->readOrError(); + if (!CounterValueOrErr) + return CounterValueOrErr.takeError(); + CounterValues = std::move(*CounterValueOrErr); + return Error::success(); } // The child exited, but not successfully diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp index 3ff1745e9e06..314de1ec3236 100644 --- a/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp +++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.cpp @@ -148,17 +148,6 @@ void Counter::stop() { ioctl(FileDescriptor, PERF_EVENT_IOC_DISABLE, 0); } -int64_t Counter::read() const { - auto ValueOrError = readOrError(); - if (ValueOrError) { - if (!ValueOrError.get().empty()) - return ValueOrError.get()[0]; - errs() << "Counter has no reading\n"; - } else - errs() << ValueOrError.takeError() << "\n"; - return -1; -} - llvm::Expected> Counter::readOrError(StringRef /*unused*/) const { int64_t Count = 0; @@ -187,8 +176,6 @@ void Counter::start() {} void Counter::stop() {} -int64_t Counter::read() const { return 42; } - llvm::Expected> Counter::readOrError(StringRef /*unused*/) const { if (IsDummyEvent) { diff --git a/llvm/tools/llvm-exegesis/lib/PerfHelper.h b/llvm/tools/llvm-exegesis/lib/PerfHelper.h index a50974f0a67b..894aac1f197e 100644 --- a/llvm/tools/llvm-exegesis/lib/PerfHelper.h +++ b/llvm/tools/llvm-exegesis/lib/PerfHelper.h @@ -95,9 +95,6 @@ public: /// Stops the measurement of the event. void stop(); - /// Returns the current value of the counter or -1 if it cannot be read. - int64_t read() const; - /// Returns the current value of the counter or error if it cannot be read. /// FunctionBytes: The benchmark function being executed. /// This is used to filter out the measurements to ensure they are only -- GitLab From d9c8edf08afce3d1e563e4521ae847a6809bb993 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 5 Jan 2024 13:38:05 -0800 Subject: [PATCH 154/728] [llvm-exegesis] Add matcher for register initial values (#76666) Currently, the unit tests for the BenchmarkResult struct do not check if the register initial values can be parsed back in. This patch adds a matcher and tests that the register initial values can be parsed correctly. This exercises code already contained within the benchmark to yaml infrastructure. --- .../tools/llvm-exegesis/X86/BenchmarkResultTest.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp index 6c558b59be98..616f7bac54bc 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp @@ -46,6 +46,15 @@ MATCHER(EqMCInst, "") { return true; } +MATCHER(EqRegValue, "") { + const RegisterValue Lhs = get<0>(arg); + const RegisterValue Rhs = get<1>(arg); + if (Lhs.Register != Rhs.Register || Lhs.Value != Rhs.Value) + return false; + + return true; +} + namespace { TEST(BenchmarkResultTest, WriteToAndReadFromDisk) { @@ -120,6 +129,8 @@ TEST(BenchmarkResultTest, WriteToAndReadFromDisk) { EXPECT_THAT(FromDisk.Key.Instructions, Pointwise(EqMCInst(), ToDisk.Key.Instructions)); EXPECT_EQ(FromDisk.Key.Config, ToDisk.Key.Config); + EXPECT_THAT(FromDisk.Key.RegisterInitialValues, + Pointwise(EqRegValue(), ToDisk.Key.RegisterInitialValues)); EXPECT_EQ(FromDisk.Mode, ToDisk.Mode); EXPECT_EQ(FromDisk.CpuName, ToDisk.CpuName); EXPECT_EQ(FromDisk.LLVMTriple, ToDisk.LLVMTriple); @@ -137,6 +148,8 @@ TEST(BenchmarkResultTest, WriteToAndReadFromDisk) { EXPECT_THAT(FromDisk.Key.Instructions, Pointwise(EqMCInst(), ToDisk.Key.Instructions)); EXPECT_EQ(FromDisk.Key.Config, ToDisk.Key.Config); + EXPECT_THAT(FromDisk.Key.RegisterInitialValues, + Pointwise(EqRegValue(), ToDisk.Key.RegisterInitialValues)); EXPECT_EQ(FromDisk.Mode, ToDisk.Mode); EXPECT_EQ(FromDisk.CpuName, ToDisk.CpuName); EXPECT_EQ(FromDisk.LLVMTriple, ToDisk.LLVMTriple); -- GitLab From 853b13342a131e06d61293ec6e840642054c6c85 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Fri, 5 Jan 2024 22:41:51 +0100 Subject: [PATCH 155/728] [clang] Fix test for case-insensitive absolute includes (#76985) When CMake on Windows is told to generate the build into a directory whose real path has a different drive letter (e.g. due to a symlink), the "clang/test/Lexer/case-insensitive-include-absolute.c" test fails. That happens because because `trySimplifyPath()` in `PPDirectives.cpp` finds out there's more than a case difference between the `#include` path (containing `%/t`) and the real path, which prevents the diagnostic to fire. I thought this is only an issue on Windows due to the fact that LIT does not drag the path to the build directory through `os.path.realpath()` like it does on other systems (see `abs_path_preserve_drive()` in "llvm/utils/lit/lit/util.py"). However, even after only using `os.path.abspath()` on a Unix system, build generated into a symlinked directory tests correctly. I assume there must be something else at play, but I don't have the time to dig deeper. The fix is is fairly straightforward: use the real path in the `#include` (with `%{/t:real}`), which removes the non-case difference and unblocks the diagnostic. --- clang/test/Lexer/case-insensitive-include-absolute.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/test/Lexer/case-insensitive-include-absolute.c b/clang/test/Lexer/case-insensitive-include-absolute.c index 6247e4808c7f..ef9e131c45df 100644 --- a/clang/test/Lexer/case-insensitive-include-absolute.c +++ b/clang/test/Lexer/case-insensitive-include-absolute.c @@ -1,8 +1,8 @@ // REQUIRES: case-insensitive-filesystem // RUN: rm -rf %t && split-file %s %t -// RUN: sed "s|DIR|%/t|g" %t/tu.c.in > %t/tu.c -// RUN: %clang_cc1 -fsyntax-only %t/tu.c 2>&1 | FileCheck %s --DDIR=%/t +// RUN: sed "s|DIR|%{/t:real}|g" %t/tu.c.in > %t/tu.c +// RUN: %clang_cc1 -fsyntax-only %t/tu.c 2>&1 | FileCheck %s -DDIR=%{/t:real} //--- header.h //--- tu.c.in -- GitLab From f1d75d08adb9841dd9cebad63b76d4823ec2bdac Mon Sep 17 00:00:00 2001 From: Jonathon Penix <107437988+jonathonpenix@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:55:50 -0800 Subject: [PATCH 156/728] [clang][Driver] Don't warn when -nostdinc and -nostdinc++ are both specified (#77130) When -nostdinc and -nostdinc++ are both specified and the Baremetal toolchain is used, an unused command line argument warning for -nostdinc++ is produced. This doesn't seem particularly meaningful as -nostdinc++ would have been claimed/used had the check in AddClangCXXStdlibIncludeArgs not short-circuited. So, just claim all arguments in this check. I believe this is consistent with what was done for the GNU toolchain (see 6fe7de90b9e4e466a5c2baadafd5f72d3203651d), so hopefully this is appropriate here as well. --- clang/lib/Driver/ToolChains/BareMetal.cpp | 5 ++--- clang/test/Driver/nostdincxx.cpp | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp index 42c8336e626c..391c47f88bde 100644 --- a/clang/lib/Driver/ToolChains/BareMetal.cpp +++ b/clang/lib/Driver/ToolChains/BareMetal.cpp @@ -293,9 +293,8 @@ void BareMetal::addClangTargetOptions(const ArgList &DriverArgs, void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs, ArgStringList &CC1Args) const { - if (DriverArgs.hasArg(options::OPT_nostdinc) || - DriverArgs.hasArg(options::OPT_nostdlibinc) || - DriverArgs.hasArg(options::OPT_nostdincxx)) + if (DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc, + options::OPT_nostdincxx)) return; const Driver &D = getDriver(); diff --git a/clang/test/Driver/nostdincxx.cpp b/clang/test/Driver/nostdincxx.cpp index ef5702a19c03..94d74f230eb1 100644 --- a/clang/test/Driver/nostdincxx.cpp +++ b/clang/test/Driver/nostdincxx.cpp @@ -2,6 +2,7 @@ // RUN: not %clangxx -nostdinc++ %s 2>&1 | FileCheck %s // RUN: not %clangxx -nostdlibinc %s 2>&1 | FileCheck %s // RUN: not %clangxx --target=x86_64-unknown-unknown-gnu -fsyntax-only -nostdinc -nostdinc++ %s 2>&1 | FileCheck /dev/null --implicit-check-not=-Wunused-command-line-argument +// RUN: not %clangxx --target=riscv64-unknown-elf -fsyntax-only -nostdinc -nostdinc++ %s 2>&1 | FileCheck /dev/null --implicit-check-not=-Wunused-command-line-argument // CHECK: 'vector' file not found #include -- GitLab From d7b4debf98fd740f821bda717de7b807e26ae95a Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 5 Jan 2024 22:13:25 +0000 Subject: [PATCH 157/728] [cmake,Apple] Check ld-classic when looking for ld64. (#77136) In newer SDKs, ld64 is called ld-classic. Look for it first, as ld in those SDKs is still missing some functionality some tests depend on. This enables running the tests from check-llvm-tools-lto with newer SDKs on ARM64 macOS. --- llvm/cmake/config-ix.cmake | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake index 3e12213f6e6b..bf1b110245bb 100644 --- a/llvm/cmake/config-ix.cmake +++ b/llvm/cmake/config-ix.cmake @@ -639,12 +639,25 @@ if(CMAKE_HOST_APPLE AND APPLE) if(NOT CMAKE_XCRUN) find_program(CMAKE_XCRUN NAMES xcrun) endif() + + # First, check if there's ld-classic, which is ld64 in newer SDKs. if(CMAKE_XCRUN) - execute_process(COMMAND ${CMAKE_XCRUN} -find ld + execute_process(COMMAND ${CMAKE_XCRUN} -find ld-classic OUTPUT_VARIABLE LD64_EXECUTABLE OUTPUT_STRIP_TRAILING_WHITESPACE) else() - find_program(LD64_EXECUTABLE NAMES ld DOC "The ld64 linker") + find_program(LD64_EXECUTABLE NAMES ld-classic DOC "The ld64 linker") + endif() + + # Otherwise look for ld directly. + if(NOT LD64_EXECUTABLE) + if(CMAKE_XCRUN) + execute_process(COMMAND ${CMAKE_XCRUN} -find ld + OUTPUT_VARIABLE LD64_EXECUTABLE + OUTPUT_STRIP_TRAILING_WHITESPACE) + else() + find_program(LD64_EXECUTABLE NAMES ld DOC "The ld64 linker") + endif() endif() endif() -- GitLab From 8e92d9ea8dfcc49223e4ac62c998aea4d2d4fc84 Mon Sep 17 00:00:00 2001 From: Shubham Sandeep Rastogi Date: Fri, 5 Jan 2024 14:22:29 -0800 Subject: [PATCH 158/728] Move verify-no-file.yaml to AArch64 folder (#77149) This test fails on power pc and solaris, moving it to AArch64 folder because the yaml is generated for an AArch64 MachO object file. --- .../test/tools/llvm-dwarfdump/{ => AArch64}/verify-no-file.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename llvm/test/tools/llvm-dwarfdump/{ => AArch64}/verify-no-file.yaml (99%) diff --git a/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml b/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml similarity index 99% rename from llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml rename to llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml index fc25fceef764..1327cc26ee9f 100644 --- a/llvm/test/tools/llvm-dwarfdump/verify-no-file.yaml +++ b/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml @@ -1,5 +1,5 @@ # RUN: yaml2obj %s -o %t.o -# RUN: llvm-dwarfdump --debug-line --verify %t.o 2>&1 | FileCheck %s +# RUN: llvm-dwarfdump -arch arm64 --debug-line --verify %t.o 2>&1 | FileCheck %s # CHECK-NOT: error: .debug_line[0x{{[0-9a-f]+}}][0] has invalid file index 1 (valid values are [1,0]): --- !mach-o -- GitLab From 5b54dd4f5eccc66c6352f9277d0a6edbd2e80435 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 5 Jan 2024 14:30:52 -0800 Subject: [PATCH 159/728] [tsan] Generalize FrameIsInternal (#77143) Prepare to move this into sanitizer_common. --- compiler-rt/lib/tsan/rtl/tsan_report.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp index 35cb6710a54f..167e4be4fc0e 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp @@ -278,13 +278,9 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { return false; const char *file = frame->info.file; const char *module = frame->info.module; - if (file != 0 && - (internal_strstr(file, "tsan_interceptors_posix.cpp") || - internal_strstr(file, "tsan_interceptors_memintrinsics.cpp") || - internal_strstr(file, "sanitizer_common_interceptors.inc") || - internal_strstr(file, "tsan_interface_"))) + if (file != 0 && (internal_strstr(file, "/compiler-rt/lib/"))) return true; - if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_"))) + if (module != 0 && (internal_strstr(module, "libclang_rt."))) return true; return false; } -- GitLab From 9cf4f10609435bccd5fb32b979557a9e339563b6 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 5 Jan 2024 14:32:18 -0800 Subject: [PATCH 160/728] [NFC][tsan] `ptr != 0` to implicit check (#77144) --- compiler-rt/lib/tsan/rtl/tsan_report.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp index 167e4be4fc0e..cdcc20b9758f 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp @@ -274,13 +274,13 @@ static ReportStack *ChooseSummaryStack(const ReportDesc *rep) { } static bool FrameIsInternal(const SymbolizedStack *frame) { - if (frame == 0) + if (!frame) return false; const char *file = frame->info.file; const char *module = frame->info.module; - if (file != 0 && (internal_strstr(file, "/compiler-rt/lib/"))) + if (file && (internal_strstr(file, "/compiler-rt/lib/"))) return true; - if (module != 0 && (internal_strstr(module, "libclang_rt."))) + if (module && (internal_strstr(module, "libclang_rt."))) return true; return false; } -- GitLab From 77724d5f46efea1306e036aa0eca8e2ecb64916c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 5 Jan 2024 22:33:37 +0000 Subject: [PATCH 161/728] [LTO] Add test showing lto-discard-value-names option being ignored. At the moment, lto-discard-value-names isn't handled properly by libLTO. --- llvm/test/tools/lto/discard-value-names.ll | 37 ++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 llvm/test/tools/lto/discard-value-names.ll diff --git a/llvm/test/tools/lto/discard-value-names.ll b/llvm/test/tools/lto/discard-value-names.ll new file mode 100644 index 000000000000..7e2c82119028 --- /dev/null +++ b/llvm/test/tools/lto/discard-value-names.ll @@ -0,0 +1,37 @@ +; RUN: llvm-as %s -o %t.o +; RUN: %ld64 -lto_library %llvmshlibdir/libLTO.dylib -dylib -arch x86_64 -macos_version_min 10.10.0 -o %t.dylib %t.o -save-temps -undefined dynamic_lookup -exported_symbol _bar -lSystem -mllvm -lto-discard-value-names +; RUN: llvm-dis %t.dylib.lto.opt.bc -o - | FileCheck --check-prefix=DISCARD %s + +; RUN: %ld64 -lto_library %llvmshlibdir/libLTO.dylib -dylib -arch x86_64 -macos_version_min 10.10.0 -o %t.dylib %t.o -save-temps -undefined dynamic_lookup -exported_symbol _bar -lSystem -mllvm -lto-discard-value-names=false +; RUN: llvm-dis %t.dylib.lto.opt.bc -o - | FileCheck --check-prefix=KEEP %s + +; FIXME: -lto-discard-value-names is ignored at the moment. +; DISCARD: %cmp.i = icmp +; DISCARD: %add = add i32 + +; KEEP: %cmp.i = icmp +; KEEP : %add = add i32 + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.10.0" + +declare void @external() + +define internal i32 @foo(i32 %a, i32 %b) { +entry: + %cmp = icmp ult i32 %a, %b + br i1 %cmp, label %then, label %else + +then: + call void @external() + ret i32 10 + +else: + ret i32 20 +} + +define i32 @bar(i32 %a) { + %res = call i32 @foo(i32 %a, i32 10) + %add = add i32 %res, %a + ret i32 %add +} -- GitLab From f74ce000752d565dbd9d6fcc1e2ed7f49ff7e398 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 5 Jan 2024 14:48:37 -0800 Subject: [PATCH 162/728] [tsan] Fallback to top frame (#77145) Probably it's not important, as it should not happen often, but if all frames are internal we should pick top, not the bottom frame. --- compiler-rt/lib/tsan/rtl/tsan_report.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp index cdcc20b9758f..c6b764bd8917 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp @@ -275,7 +275,7 @@ static ReportStack *ChooseSummaryStack(const ReportDesc *rep) { static bool FrameIsInternal(const SymbolizedStack *frame) { if (!frame) - return false; + return true; const char *file = frame->info.file; const char *module = frame->info.module; if (file && (internal_strstr(file, "/compiler-rt/lib/"))) @@ -286,9 +286,10 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { } static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) { - while (FrameIsInternal(frames) && frames->next) - frames = frames->next; - return frames; + for (SymbolizedStack *f = frames; f; f = f->next) + if (!FrameIsInternal(f)) + return f; + return frames; // Fallback to the top frame. } void PrintReport(const ReportDesc *rep) { -- GitLab From a89141f733cef817c586bb6da0ea69a5a323874e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danny=20M=C3=B6sch?= Date: Sat, 6 Jan 2024 00:14:08 +0100 Subject: [PATCH 163/728] [clang-tidy] Add check `readability-avoid-return-with-void-value` (#76249) --- .../AvoidReturnWithVoidValueCheck.cpp | 57 +++++++++++++++ .../AvoidReturnWithVoidValueCheck.h | 44 ++++++++++++ .../clang-tidy/readability/CMakeLists.txt | 1 + .../readability/ReadabilityTidyModule.cpp | 3 + clang-tools-extra/docs/ReleaseNotes.rst | 6 ++ .../docs/clang-tidy/checks/list.rst | 1 + .../avoid-return-with-void-value.rst | 51 ++++++++++++++ .../avoid-return-with-void-value.cpp | 69 +++++++++++++++++++ 8 files changed, 232 insertions(+) create mode 100644 clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.cpp create mode 100644 clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.h create mode 100644 clang-tools-extra/docs/clang-tidy/checks/readability/avoid-return-with-void-value.rst create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/avoid-return-with-void-value.cpp diff --git a/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.cpp b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.cpp new file mode 100644 index 000000000000..e3400f614fa5 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.cpp @@ -0,0 +1,57 @@ +//===--- AvoidReturnWithVoidValueCheck.cpp - clang-tidy -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AvoidReturnWithVoidValueCheck.h" +#include "clang/AST/Stmt.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +static constexpr auto IgnoreMacrosName = "IgnoreMacros"; +static constexpr auto IgnoreMacrosDefault = true; + +static constexpr auto StrictModeName = "StrictMode"; +static constexpr auto StrictModeDefault = true; + +AvoidReturnWithVoidValueCheck::AvoidReturnWithVoidValueCheck( + StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context), + IgnoreMacros( + Options.getLocalOrGlobal(IgnoreMacrosName, IgnoreMacrosDefault)), + StrictMode(Options.getLocalOrGlobal(StrictModeName, StrictModeDefault)) {} + +void AvoidReturnWithVoidValueCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher( + returnStmt( + hasReturnValue(allOf(hasType(voidType()), unless(initListExpr()))), + optionally(hasParent(compoundStmt().bind("compound_parent")))) + .bind("void_return"), + this); +} + +void AvoidReturnWithVoidValueCheck::check( + const MatchFinder::MatchResult &Result) { + const auto *VoidReturn = Result.Nodes.getNodeAs("void_return"); + if (IgnoreMacros && VoidReturn->getBeginLoc().isMacroID()) + return; + if (!StrictMode && !Result.Nodes.getNodeAs("compound_parent")) + return; + diag(VoidReturn->getBeginLoc(), "return statement within a void function " + "should not have a specified return value"); +} + +void AvoidReturnWithVoidValueCheck::storeOptions( + ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, IgnoreMacrosName, IgnoreMacros); + Options.store(Opts, StrictModeName, StrictMode); +} + +} // namespace clang::tidy::readability diff --git a/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.h b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.h new file mode 100644 index 000000000000..f8148db43cd9 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.h @@ -0,0 +1,44 @@ +//===--- AvoidReturnWithVoidValueCheck.h - clang-tidy -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOIDRETURNWITHVOIDVALUECHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOIDRETURNWITHVOIDVALUECHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Finds return statements with `void` values used within functions with `void` +/// result types. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/readability/avoid-return-with-void-value.html +class AvoidReturnWithVoidValueCheck : public ClangTidyCheck { +public: + AvoidReturnWithVoidValueCheck(StringRef Name, ClangTidyContext *Context); + + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + +private: + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } + bool isLanguageVersionSupported(const LangOptions &LangOpts) const override { + return LangOpts.CPlusPlus; + } + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; + +private: + bool IgnoreMacros; + bool StrictMode; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOIDRETURNWITHVOIDVALUECHECK_H diff --git a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt index 5452c2d48a46..408c822b861c 100644 --- a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt @@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS add_clang_library(clangTidyReadabilityModule AvoidConstParamsInDecls.cpp + AvoidReturnWithVoidValueCheck.cpp AvoidUnconditionalPreprocessorIfCheck.cpp BracesAroundStatementsCheck.cpp ConstReturnTypeCheck.cpp diff --git a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp index b8e6e6414320..0b0aad7c0dcb 100644 --- a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp @@ -10,6 +10,7 @@ #include "../ClangTidyModule.h" #include "../ClangTidyModuleRegistry.h" #include "AvoidConstParamsInDecls.h" +#include "AvoidReturnWithVoidValueCheck.h" #include "AvoidUnconditionalPreprocessorIfCheck.h" #include "BracesAroundStatementsCheck.h" #include "ConstReturnTypeCheck.h" @@ -63,6 +64,8 @@ public: void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override { CheckFactories.registerCheck( "readability-avoid-const-params-in-decls"); + CheckFactories.registerCheck( + "readability-avoid-return-with-void-value"); CheckFactories.registerCheck( "readability-avoid-unconditional-preprocessor-if"); CheckFactories.registerCheck( diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 4d25e2ebe85f..08ade306b5a0 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -230,6 +230,12 @@ New checks Detects C++ code where a reference variable is used to extend the lifetime of a temporary object that has just been constructed. +- New :doc:`readability-avoid-return-with-void-value + ` check. + + Finds return statements with ``void`` values used within functions with + ``void`` result types. + New check aliases ^^^^^^^^^^^^^^^^^ diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst index b36bf7d497b9..2f86121ad872 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/list.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst @@ -337,6 +337,7 @@ Clang-Tidy Checks :doc:`portability-simd-intrinsics `, :doc:`portability-std-allocator-const `, :doc:`readability-avoid-const-params-in-decls `, "Yes" + :doc:`readability-avoid-return-with-void-value `, :doc:`readability-avoid-unconditional-preprocessor-if `, :doc:`readability-braces-around-statements `, "Yes" :doc:`readability-const-return-type `, "Yes" diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-return-with-void-value.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-return-with-void-value.rst new file mode 100644 index 000000000000..d802f9be829c --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-return-with-void-value.rst @@ -0,0 +1,51 @@ +.. title:: clang-tidy - readability-avoid-return-with-void-value + +readability-avoid-return-with-void-value +======================================== + +Finds return statements with ``void`` values used within functions with +``void`` result types. + +A function with a ``void`` return type is intended to perform a task without +producing a return value. Return statements with expressions could lead +to confusion and may miscommunicate the function's intended behavior. + +Example: + +.. code-block:: + + void g(); + void f() { + // ... + return g(); + } + +In a long function body, the ``return`` statement suggests that the function +returns a value. However, ``return g();`` is a combination of two statements +that should be written as + +.. code-block:: + + g(); + return; + +to make clear that ``g()`` is called and immediately afterwards the function +returns (nothing). + +In C, the same issue is detected by the compiler if the ``-Wpedantic`` mode +is enabled. + +Options +------- + +.. option:: IgnoreMacros + + The value `false` specifies that return statements expanded + from macros are not checked. The default value is `true`. + +.. option:: StrictMode + + The value `false` specifies that a direct return statement shall + be excluded from the analysis if it is the only statement not + contained in a block like ``if (cond) return g();``. The default + value is `true`. diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-return-with-void-value.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-return-with-void-value.cpp new file mode 100644 index 000000000000..f00407c99ce5 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-return-with-void-value.cpp @@ -0,0 +1,69 @@ +// RUN: %check_clang_tidy %s readability-avoid-return-with-void-value %t +// RUN: %check_clang_tidy -check-suffixes=,INCLUDE-MACROS %s readability-avoid-return-with-void-value %t \ +// RUN: -- -config="{CheckOptions: [{key: readability-avoid-return-with-void-value.IgnoreMacros, value: false}]}" \ +// RUN: -- +// RUN: %check_clang_tidy -check-suffixes=LENIENT %s readability-avoid-return-with-void-value %t \ +// RUN: -- -config="{CheckOptions: [{key: readability-avoid-return-with-void-value.StrictMode, value: false}]}" \ +// RUN: -- + +void f1(); + +void f2() { + return f1(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +void f3(bool b) { + if (b) return f1(); + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + return f2(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +template +T f4() {} + +void f5() { + return f4(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +void f6() { return; } + +int f7() { return 1; } + +int f8() { return f7(); } + +void f9() { + return (void)f7(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +#define RETURN_VOID return (void)1 + +void f10() { + RETURN_VOID; + // CHECK-MESSAGES-INCLUDE-MACROS: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +template +struct C { + C(A) {} +}; + +template +C f11() { return {}; } + +using VOID = void; + +VOID f12(); + +VOID f13() { + return f12(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} -- GitLab From a9c5bddc8f18926bac6dc224144a32512207bd38 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Fri, 5 Jan 2024 23:14:39 +0000 Subject: [PATCH 164/728] [gn build] Port a89141f733ce --- .../secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn index 56d2c25e06d8..ad4de05f136f 100644 --- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn +++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn @@ -13,6 +13,7 @@ static_library("readability") { ] sources = [ "AvoidConstParamsInDecls.cpp", + "AvoidReturnWithVoidValueCheck.cpp", "AvoidUnconditionalPreprocessorIfCheck.cpp", "BracesAroundStatementsCheck.cpp", "ConstReturnTypeCheck.cpp", -- GitLab From 08c5f1fede969e687e77d0508008682e5f188f49 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Fri, 5 Jan 2024 15:32:29 -0800 Subject: [PATCH 165/728] [ORC] Add absoluteSymbolsLinkGraph to expose absolute symbols to platform (#77008) Adds a function to create a LinkGraph of absolute symbols, and a callback in dynamic library search generators to enable using it to expose its symbols to the platform/orc runtime. This allows e.g. using __orc_rt_run_program to run a precompiled function that was found via dlsym. Ideally we would use this in llvm-jitlink's own search generator, but it will require more work to align with the Process/Platform JITDylib split, so not handled here. As part of this change we need to handle LinkGraphs that only have absolute symbols. --- bolt/lib/Rewrite/JITLinkLinker.cpp | 3 +- .../llvm/ExecutionEngine/JITLink/JITLink.h | 5 + .../Orc/EPCDynamicLibrarySearchGenerator.h | 23 +++-- .../llvm/ExecutionEngine/Orc/ExecutionUtils.h | 20 +++- llvm/lib/ExecutionEngine/JITLink/JITLink.cpp | 35 +++++++ .../JITLink/JITLinkGeneric.cpp | 14 +++ .../Orc/EPCDynamicLibrarySearchGenerator.cpp | 12 ++- .../ExecutionEngine/Orc/ExecutionUtils.cpp | 12 ++- .../lib/ExecutionEngine/Orc/MachOPlatform.cpp | 2 + .../Orc/ObjectLinkingLayer.cpp | 14 ++- .../Orc/ObjectLinkingLayerTest.cpp | 97 +++++++++++++++++++ 11 files changed, 213 insertions(+), 24 deletions(-) diff --git a/bolt/lib/Rewrite/JITLinkLinker.cpp b/bolt/lib/Rewrite/JITLinkLinker.cpp index 10a3b6a0407f..66e129bf1d05 100644 --- a/bolt/lib/Rewrite/JITLinkLinker.cpp +++ b/bolt/lib/Rewrite/JITLinkLinker.cpp @@ -173,7 +173,8 @@ struct JITLinkLinker::Context : jitlink::JITLinkContext { void notifyFinalized( jitlink::JITLinkMemoryManager::FinalizedAlloc Alloc) override { - Linker.Allocs.push_back(std::move(Alloc)); + if (Alloc) + Linker.Allocs.push_back(std::move(Alloc)); ++Linker.MM->ObjectsLoaded; } }; diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 8a019492c12d..f4d2f56c34d9 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -19,6 +19,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" #include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h" @@ -1923,6 +1924,10 @@ void visitExistingEdges(LinkGraph &G, VisitorTs &&...Vs) { Expected> createLinkGraphFromObject(MemoryBufferRef ObjectBuffer); +/// Create a \c LinkGraph defining the given absolute symbols. +std::unique_ptr absoluteSymbolsLinkGraph(const Triple &TT, + orc::SymbolMap Symbols); + /// Link the given graph. void link(std::unique_ptr G, std::unique_ptr Ctx); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h index 63797edec89e..e56afe4fe656 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h @@ -25,6 +25,7 @@ class ExecutorProcessControl; class EPCDynamicLibrarySearchGenerator : public DefinitionGenerator { public: using SymbolPredicate = unique_function; + using AddAbsoluteSymbolsFn = unique_function; /// Create a DynamicLibrarySearchGenerator that searches for symbols in the /// library with the given handle. @@ -32,24 +33,31 @@ public: /// If the Allow predicate is given then only symbols matching the predicate /// will be searched for. If the predicate is not given then all symbols will /// be searched for. - EPCDynamicLibrarySearchGenerator(ExecutionSession &ES, - tpctypes::DylibHandle H, - SymbolPredicate Allow = SymbolPredicate()) - : EPC(ES.getExecutorProcessControl()), H(H), Allow(std::move(Allow)) {} + /// + /// If \p AddAbsoluteSymbols is provided, it is used to add the symbols to the + /// \c JITDylib; otherwise it uses JD.define(absoluteSymbols(...)). + EPCDynamicLibrarySearchGenerator( + ExecutionSession &ES, tpctypes::DylibHandle H, + SymbolPredicate Allow = SymbolPredicate(), + AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr) + : EPC(ES.getExecutorProcessControl()), H(H), Allow(std::move(Allow)), + AddAbsoluteSymbols(std::move(AddAbsoluteSymbols)) {} /// Permanently loads the library at the given path and, on success, returns /// a DynamicLibrarySearchGenerator that will search it for symbol definitions /// in the library. On failure returns the reason the library failed to load. static Expected> Load(ExecutionSession &ES, const char *LibraryPath, - SymbolPredicate Allow = SymbolPredicate()); + SymbolPredicate Allow = SymbolPredicate(), + AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr); /// Creates a EPCDynamicLibrarySearchGenerator that searches for symbols in /// the target process. static Expected> GetForTargetProcess(ExecutionSession &ES, - SymbolPredicate Allow = SymbolPredicate()) { - return Load(ES, nullptr, std::move(Allow)); + SymbolPredicate Allow = SymbolPredicate(), + AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr) { + return Load(ES, nullptr, std::move(Allow), std::move(AddAbsoluteSymbols)); } Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, @@ -60,6 +68,7 @@ private: ExecutorProcessControl &EPC; tpctypes::DylibHandle H; SymbolPredicate Allow; + AddAbsoluteSymbolsFn AddAbsoluteSymbols; }; } // end namespace orc diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index 6a43376a5bd9..f7c286bec778 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -216,6 +216,7 @@ private: class DynamicLibrarySearchGenerator : public DefinitionGenerator { public: using SymbolPredicate = std::function; + using AddAbsoluteSymbolsFn = unique_function; /// Create a DynamicLibrarySearchGenerator that searches for symbols in the /// given sys::DynamicLibrary. @@ -223,22 +224,30 @@ public: /// If the Allow predicate is given then only symbols matching the predicate /// will be searched for. If the predicate is not given then all symbols will /// be searched for. - DynamicLibrarySearchGenerator(sys::DynamicLibrary Dylib, char GlobalPrefix, - SymbolPredicate Allow = SymbolPredicate()); + /// + /// If \p AddAbsoluteSymbols is provided, it is used to add the symbols to the + /// \c JITDylib; otherwise it uses JD.define(absoluteSymbols(...)). + DynamicLibrarySearchGenerator( + sys::DynamicLibrary Dylib, char GlobalPrefix, + SymbolPredicate Allow = SymbolPredicate(), + AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr); /// Permanently loads the library at the given path and, on success, returns /// a DynamicLibrarySearchGenerator that will search it for symbol definitions /// in the library. On failure returns the reason the library failed to load. static Expected> Load(const char *FileName, char GlobalPrefix, - SymbolPredicate Allow = SymbolPredicate()); + SymbolPredicate Allow = SymbolPredicate(), + AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr); /// Creates a DynamicLibrarySearchGenerator that searches for symbols in /// the current process. static Expected> GetForCurrentProcess(char GlobalPrefix, - SymbolPredicate Allow = SymbolPredicate()) { - return Load(nullptr, GlobalPrefix, std::move(Allow)); + SymbolPredicate Allow = SymbolPredicate(), + AddAbsoluteSymbolsFn AddAbsoluteSymbols = nullptr) { + return Load(nullptr, GlobalPrefix, std::move(Allow), + std::move(AddAbsoluteSymbols)); } Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, @@ -248,6 +257,7 @@ public: private: sys::DynamicLibrary Dylib; SymbolPredicate Allow; + AddAbsoluteSymbolsFn AddAbsoluteSymbols; char GlobalPrefix; }; diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index d86ceb99ded0..7f743dba60a9 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -468,6 +468,41 @@ createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) { }; } +std::unique_ptr absoluteSymbolsLinkGraph(const Triple &TT, + orc::SymbolMap Symbols) { + unsigned PointerSize; + endianness Endianness = + TT.isLittleEndian() ? endianness::little : endianness::big; + switch (TT.getArch()) { + case Triple::aarch64: + case llvm::Triple::riscv64: + case Triple::x86_64: + PointerSize = 8; + break; + case llvm::Triple::arm: + case llvm::Triple::riscv32: + case llvm::Triple::x86: + PointerSize = 4; + break; + default: + llvm::report_fatal_error("unhandled target architecture"); + } + + static std::atomic Counter = {0}; + auto Index = Counter.fetch_add(1, std::memory_order_relaxed); + auto G = std::make_unique( + "", TT, PointerSize, + Endianness, /*GetEdgeKindName=*/nullptr); + for (auto &[Name, Def] : Symbols) { + auto &Sym = + G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0, + Linkage::Strong, Scope::Default, /*IsLive=*/true); + Sym.setCallable(Def.getFlags().isCallable()); + } + + return G; +} + void link(std::unique_ptr G, std::unique_ptr Ctx) { switch (G->getTargetTriple().getObjectFormat()) { case Triple::MachO: diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 5361272ae79e..01144763ac4c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -48,6 +48,14 @@ void JITLinkerBase::linkPhase1(std::unique_ptr Self) { if (auto Err = runPasses(Passes.PostPrunePasses)) return Ctx->notifyFailed(std::move(Err)); + // Skip straight to phase 2 if the graph is empty with no associated actions. + if (G->allocActions().empty() && llvm::all_of(G->sections(), [](Section &S) { + return S.getMemLifetime() == orc::MemLifetime::NoAlloc; + })) { + linkPhase2(std::move(Self), nullptr); + return; + } + Ctx->getMemoryManager().allocate( Ctx->getJITLinkDylib(), *G, [S = std::move(Self)](AllocResult AR) mutable { @@ -163,6 +171,12 @@ void JITLinkerBase::linkPhase3(std::unique_ptr Self, if (auto Err = runPasses(Passes.PostFixupPasses)) return abandonAllocAndBailOut(std::move(Self), std::move(Err)); + // Skip straight to phase 4 if the graph has no allocation. + if (!Alloc) { + linkPhase4(std::move(Self), JITLinkMemoryManager::FinalizedAlloc{}); + return; + } + Alloc->finalize([S = std::move(Self)](FinalizeResult FR) mutable { // FIXME: Once MSVC implements c++17 order of evaluation rules for calls // this can be simplified to diff --git a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp index 2e2e7a9c5f32..460f4e1c448e 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp @@ -12,15 +12,15 @@ namespace llvm { namespace orc { Expected> -EPCDynamicLibrarySearchGenerator::Load(ExecutionSession &ES, - const char *LibraryPath, - SymbolPredicate Allow) { +EPCDynamicLibrarySearchGenerator::Load( + ExecutionSession &ES, const char *LibraryPath, SymbolPredicate Allow, + AddAbsoluteSymbolsFn AddAbsoluteSymbols) { auto Handle = ES.getExecutorProcessControl().loadDylib(LibraryPath); if (!Handle) return Handle.takeError(); - return std::make_unique(ES, *Handle, - std::move(Allow)); + return std::make_unique( + ES, *Handle, std::move(Allow), std::move(AddAbsoluteSymbols)); } Error EPCDynamicLibrarySearchGenerator::tryToGenerate( @@ -62,6 +62,8 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate( return Error::success(); // Define resolved symbols. + if (AddAbsoluteSymbols) + return AddAbsoluteSymbols(JD, std::move(NewSymbols)); return JD.define(absoluteSymbols(std::move(NewSymbols))); } diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 8d5608cc4d4c..3952445bb1aa 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -218,19 +218,23 @@ void ItaniumCXAAtExitSupport::runAtExits(void *DSOHandle) { } DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( - sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow, + AddAbsoluteSymbolsFn AddAbsoluteSymbols) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), + AddAbsoluteSymbols(std::move(AddAbsoluteSymbols)), GlobalPrefix(GlobalPrefix) {} Expected> DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, - SymbolPredicate Allow) { + SymbolPredicate Allow, + AddAbsoluteSymbolsFn AddAbsoluteSymbols) { std::string ErrMsg; auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); if (!Lib.isValid()) return make_error(std::move(ErrMsg), inconvertibleErrorCode()); return std::make_unique( - std::move(Lib), GlobalPrefix, std::move(Allow)); + std::move(Lib), GlobalPrefix, std::move(Allow), + std::move(AddAbsoluteSymbols)); } Error DynamicLibrarySearchGenerator::tryToGenerate( @@ -261,6 +265,8 @@ Error DynamicLibrarySearchGenerator::tryToGenerate( if (NewSymbols.empty()) return Error::success(); + if (AddAbsoluteSymbols) + return AddAbsoluteSymbols(JD, std::move(NewSymbols)); return JD.define(absoluteSymbols(std::move(NewSymbols))); } diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 9057300bf043..6c17f14aa4c7 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -1608,6 +1608,8 @@ Error MachOPlatform::MachOPlatformPlugin::prepareSymbolTableRegistration( SmallVector SymsToProcess; for (auto *Sym : G.defined_symbols()) SymsToProcess.push_back(Sym); + for (auto *Sym : G.absolute_symbols()) + SymsToProcess.push_back(Sym); for (auto *Sym : SymsToProcess) { if (!Sym->hasName()) diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 3d77f82e6569..b8282948034e 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -93,15 +93,20 @@ private: Interface LGI; - for (auto *Sym : G.defined_symbols()) { + auto AddSymbol = [&](Symbol *Sym) { // Skip local symbols. if (Sym->getScope() == Scope::Local) - continue; + return; assert(Sym->hasName() && "Anonymous non-local symbol?"); LGI.SymbolFlags[ES.intern(Sym->getName())] = getJITSymbolFlagsForSymbol(*Sym); - } + }; + + for (auto *Sym : G.defined_symbols()) + AddSymbol(Sym); + for (auto *Sym : G.absolute_symbols()) + AddSymbol(Sym); if (hasInitializerSection(G)) LGI.InitSymbol = makeInitSymbol(ES, G); @@ -705,6 +710,9 @@ Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, if (Err) return Err; + if (!FA) + return Error::success(); + return MR.withResourceKeyDo( [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); } diff --git a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp index 91659240c9d6..edd12ebb62e1 100644 --- a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp @@ -10,6 +10,11 @@ #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/Testing/Support/Error.h" #include "gtest/gtest.h" @@ -173,4 +178,96 @@ TEST_F(ObjectLinkingLayerTest, HandleErrorDuringPostAllocationPass) { EXPECT_THAT_EXPECTED(ES.lookup(&JD, "_anchor"), Failed()); } +TEST(ObjectLinkingLayerSearchGeneratorTest, AbsoluteSymbolsObjectLayer) { + class TestEPC : public UnsupportedExecutorProcessControl { + public: + TestEPC() + : UnsupportedExecutorProcessControl(nullptr, nullptr, + "x86_64-apple-darwin") {} + + Expected loadDylib(const char *DylibPath) override { + return ExecutorAddr::fromPtr((void *)nullptr); + } + + Expected> + lookupSymbols(ArrayRef Request) override { + std::vector Result; + EXPECT_EQ(Request.size(), 1u); + for (auto &LR : Request) { + EXPECT_EQ(LR.Symbols.size(), 1u); + for (auto &Sym : LR.Symbols) { + if (*Sym.first == "_testFunc") { + ExecutorSymbolDef Def{ExecutorAddr::fromPtr((void *)0x1000), + JITSymbolFlags::Exported}; + Result.push_back(Def); + } else { + ADD_FAILURE() << "unexpected symbol request " << *Sym.first; + } + } + } + return std::vector{1, Result}; + } + }; + + ExecutionSession ES{std::make_unique()}; + JITDylib &JD = ES.createBareJITDylib("main"); + ObjectLinkingLayer ObjLinkingLayer{ + ES, std::make_unique(4096)}; + + auto G = EPCDynamicLibrarySearchGenerator::GetForTargetProcess( + ES, {}, [&](JITDylib &JD, SymbolMap Syms) { + auto G = + absoluteSymbolsLinkGraph(ES.getTargetTriple(), std::move(Syms)); + return ObjLinkingLayer.add(JD, std::move(G)); + }); + ASSERT_THAT_EXPECTED(G, Succeeded()); + JD.addGenerator(std::move(*G)); + + class CheckDefs : public ObjectLinkingLayer::Plugin { + public: + ~CheckDefs() { EXPECT_TRUE(SawSymbolDef); } + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) override { + Config.PostAllocationPasses.push_back([this](LinkGraph &G) { + unsigned SymCount = 0; + for (Symbol *Sym : G.absolute_symbols()) { + SymCount += 1; + if (!Sym->hasName()) { + ADD_FAILURE() << "unexpected unnamed symbol"; + continue; + } + if (Sym->getName() == "_testFunc") + SawSymbolDef = true; + else + ADD_FAILURE() << "unexpected symbol " << Sym->getName(); + } + EXPECT_EQ(SymCount, 1u); + return Error::success(); + }); + } + + Error notifyFailed(MaterializationResponsibility &MR) override { + return Error::success(); + } + + Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override { + return Error::success(); + } + void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey, + ResourceKey SrcKey) override { + llvm_unreachable("unexpected resource transfer"); + } + + private: + bool SawSymbolDef = false; + }; + + ObjLinkingLayer.addPlugin(std::make_unique()); + + EXPECT_THAT_EXPECTED(ES.lookup(&JD, "_testFunc"), Succeeded()); + EXPECT_THAT_ERROR(ES.endSession(), Succeeded()); +} + } // end anonymous namespace -- GitLab From def42537ee65fd5d309b7ab70a9ea4d32eb6aadd Mon Sep 17 00:00:00 2001 From: Daniel Hoekwater Date: Tue, 19 Dec 2023 01:26:34 +0000 Subject: [PATCH 166/728] [NFC][CodeGen][AArch64] Add tests for unconditional branch duplication c9f3288 introduced unconditional branch deduplication for basic block sections and machine function splitting, but it didn't add tests for AArch64 since prior behavior crashed the test. This change adds tests for AArch64 and has no functional change. --- llvm/test/CodeGen/Generic/machine-function-splitter.ll | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/test/CodeGen/Generic/machine-function-splitter.ll b/llvm/test/CodeGen/Generic/machine-function-splitter.ll index a236f201eaaf..d7cc3941ee7a 100644 --- a/llvm/test/CodeGen/Generic/machine-function-splitter.ll +++ b/llvm/test/CodeGen/Generic/machine-function-splitter.ll @@ -12,6 +12,7 @@ ; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-AARCH64 ; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-AARCH64 ; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-AARCH64 +; RUN: llc < %s -mtriple=aarch64 -split-machine-functions -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-AARCH64 ; RUN: llc < %s -mtriple=aarch64 -enable-split-machine-functions -aarch64-redzone | FileCheck %s -check-prefixes=MFS-REDZONE-AARCH64 ; COM: Machine function splitting with AFDO profiles @@ -470,9 +471,8 @@ define void @foo16(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ; MFS-O0-LABEL: foo16 ; MFS-O0-X86: jmp ; MFS-O0-X86-NOT: jmp -; MFS-O0-AARCH64: br -; MFS-O0-AARCH64: br -; MFS-O0-AARCH64-NOT: br +; MFS-O0-AARCH64: b foo16.cold +; MFS-O0-AARCH64-NOT: b foo16.cold ; MFS-O0: .section .text.split.foo16 ; MFS-O0-NEXT: foo16.cold %2 = call i32 @baz() -- GitLab From 651a42ff65c3afc2a2ec6c7fc19009387757d686 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Fri, 5 Jan 2024 15:51:00 -0800 Subject: [PATCH 167/728] [ORC] Remove unused flag (PendingRemoval) from JITDylib symbol table. --- llvm/include/llvm/ExecutionEngine/Orc/Core.h | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index ba164c6b629e..6a9bcf712169 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -1210,14 +1210,13 @@ private: SymbolTableEntry() = default; SymbolTableEntry(JITSymbolFlags Flags) : Flags(Flags), State(static_cast(SymbolState::NeverSearched)), - MaterializerAttached(false), PendingRemoval(false) {} + MaterializerAttached(false) {} ExecutorAddr getAddress() const { return Addr; } JITSymbolFlags getFlags() const { return Flags; } SymbolState getState() const { return static_cast(State); } bool hasMaterializerAttached() const { return MaterializerAttached; } - bool isPendingRemoval() const { return PendingRemoval; } void setAddress(ExecutorAddr Addr) { this->Addr = Addr; } void setFlags(JITSymbolFlags Flags) { this->Flags = Flags; } @@ -1231,18 +1230,13 @@ private: this->MaterializerAttached = MaterializerAttached; } - void setPendingRemoval(bool PendingRemoval) { - this->PendingRemoval = PendingRemoval; - } - ExecutorSymbolDef getSymbol() const { return {Addr, Flags}; } private: ExecutorAddr Addr; JITSymbolFlags Flags; - uint8_t State : 6; + uint8_t State : 7; uint8_t MaterializerAttached : 1; - uint8_t PendingRemoval : 1; }; using SymbolTable = DenseMap; -- GitLab From 5e54319b7be3e8aa035836098e0a9defc0a41c3a Mon Sep 17 00:00:00 2001 From: Dimple Prajapati Date: Fri, 5 Jan 2024 16:27:30 -0800 Subject: [PATCH 168/728] [mlir][spirv] Support spec constants as GlobalVar initializer (#75660) Changes include: - spirv serialization and deserialization needs handling in cases when GlobalVariableOp initializer is defined using spirv SpecConstant or SpecConstantComposite op, currently even though it allows SpecConst, it only looked up in for GlobalVariable Map to find initializer symbol reference, change is fixing this and extending the support to SpecConstantComposite as an initializer. - Adds tests to make sure GlobalVariable can be initialized using specialized constants. --------- Co-authored-by: Lei Zhang --- mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp | 7 +++--- .../SPIRV/Deserialization/Deserializer.cpp | 16 +++++++++---- .../SPIRV/Serialization/SerializeOps.cpp | 23 +++++++++++++----- mlir/test/Dialect/SPIRV/IR/structure-ops.mlir | 15 +++++++++++- mlir/test/Target/SPIRV/global-variable.mlir | 24 +++++++++++++++++++ 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp index 2a1d08330828..5343a12132a9 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp @@ -1162,10 +1162,11 @@ LogicalResult spirv::GlobalVariableOp::verify() { // TODO: Currently only variable initialization with specialization // constants and other variables is supported. They could be normal // constants in the module scope as well. - if (!initOp || - !isa(initOp)) { + if (!initOp || !isa(initOp)) { return emitOpError("initializer must be result of a " - "spirv.SpecConstant or spirv.GlobalVariable op"); + "spirv.SpecConstant or spirv.GlobalVariable or " + "spirv.SpecConstantCompositeOp op"); } } diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp index 89e2e7ad52fa..00645d2c4551 100644 --- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp +++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp @@ -637,14 +637,22 @@ spirv::Deserializer::processGlobalVariable(ArrayRef operands) { // Initializer. FlatSymbolRefAttr initializer = nullptr; + if (wordIndex < operands.size()) { - auto initializerOp = getGlobalVariable(operands[wordIndex]); - if (!initializerOp) { + Operation *op = nullptr; + + if (auto initOp = getGlobalVariable(operands[wordIndex])) + op = initOp; + else if (auto initOp = getSpecConstant(operands[wordIndex])) + op = initOp; + else if (auto initOp = getSpecConstantComposite(operands[wordIndex])) + op = initOp; + else return emitError(unknownLoc, "unknown ") << operands[wordIndex] << "used as initializer"; - } + + initializer = SymbolRefAttr::get(op); wordIndex++; - initializer = SymbolRefAttr::get(initializerOp.getOperation()); } if (wordIndex != operands.size()) { return emitError(unknownLoc, diff --git a/mlir/lib/Target/SPIRV/Serialization/SerializeOps.cpp b/mlir/lib/Target/SPIRV/Serialization/SerializeOps.cpp index 44538c38a41b..7bfcca5b4dcd 100644 --- a/mlir/lib/Target/SPIRV/Serialization/SerializeOps.cpp +++ b/mlir/lib/Target/SPIRV/Serialization/SerializeOps.cpp @@ -383,20 +383,31 @@ Serializer::processGlobalVariableOp(spirv::GlobalVariableOp varOp) { operands.push_back(static_cast(varOp.storageClass())); // Encode initialization. - if (auto initializer = varOp.getInitializer()) { - auto initializerID = getVariableID(*initializer); - if (!initializerID) { + StringRef initAttrName = varOp.getInitializerAttrName().getValue(); + if (std::optional initSymbolName = varOp.getInitializer()) { + uint32_t initializerID = 0; + auto initRef = varOp->getAttrOfType(initAttrName); + Operation *initOp = SymbolTable::lookupNearestSymbolFrom( + varOp->getParentOp(), initRef.getAttr()); + + // Check if initializer is GlobalVariable or SpecConstant* cases. + if (isa(initOp)) + initializerID = getVariableID(*initSymbolName); + else + initializerID = getSpecConstID(*initSymbolName); + + if (!initializerID) return emitError(varOp.getLoc(), "invalid usage of undefined variable as initializer"); - } + operands.push_back(initializerID); - elidedAttrs.push_back("initializer"); + elidedAttrs.push_back(initAttrName); } if (failed(emitDebugLine(typesGlobalValues, varOp.getLoc()))) return failure(); encodeInstructionInto(typesGlobalValues, spirv::Opcode::OpVariable, operands); - elidedAttrs.push_back("initializer"); + elidedAttrs.push_back(initAttrName); // Encode decorations. for (auto attr : varOp->getAttrs()) { diff --git a/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir b/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir index 722e4434aeaf..77b605050e14 100644 --- a/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir +++ b/mlir/test/Dialect/SPIRV/IR/structure-ops.mlir @@ -349,6 +349,19 @@ spirv.SpecConstant @sc = 4.0 : f32 // CHECK: spirv.GlobalVariable @var initializer(@sc) spirv.GlobalVariable @var initializer(@sc) : !spirv.ptr + +// ----- +// Allow SpecConstantComposite as initializer + spirv.module Logical GLSL450 { + spirv.SpecConstant @sc1 = 1 : i8 + spirv.SpecConstant @sc2 = 2 : i8 + spirv.SpecConstant @sc3 = 3 : i8 + spirv.SpecConstantComposite @scc (@sc1, @sc2, @sc3) : !spirv.array<3 x i8> + + // CHECK: spirv.GlobalVariable @var initializer(@scc) : !spirv.ptr, Private> + spirv.GlobalVariable @var initializer(@scc) : !spirv.ptr, Private> +} + // ----- spirv.module Logical GLSL450 { @@ -410,7 +423,7 @@ spirv.module Logical GLSL450 { // ----- spirv.module Logical GLSL450 { - // expected-error @+1 {{op initializer must be result of a spirv.SpecConstant or spirv.GlobalVariable op}} + // expected-error @+1 {{op initializer must be result of a spirv.SpecConstant or spirv.GlobalVariable or spirv.SpecConstantCompositeOp op}} spirv.GlobalVariable @var0 initializer(@var1) : !spirv.ptr } diff --git a/mlir/test/Target/SPIRV/global-variable.mlir b/mlir/test/Target/SPIRV/global-variable.mlir index 66d0782c205c..f22d2a9b3d14 100644 --- a/mlir/test/Target/SPIRV/global-variable.mlir +++ b/mlir/test/Target/SPIRV/global-variable.mlir @@ -23,6 +23,30 @@ spirv.module Logical GLSL450 requires #spirv.vce { // ----- +spirv.module Logical GLSL450 requires #spirv.vce { + // CHECK: spirv.SpecConstant @sc = 1 : i8 + // CHECK-NEXT: spirv.GlobalVariable @var initializer(@sc) : !spirv.ptr + spirv.SpecConstant @sc = 1 : i8 + + spirv.GlobalVariable @var initializer(@sc) : !spirv.ptr +} + +// ----- + +spirv.module Logical GLSL450 requires #spirv.vce { + // CHECK: spirv.SpecConstantComposite @scc (@sc0, @sc1, @sc2) : !spirv.array<3 x i8> + // CHECK-NEXT: spirv.GlobalVariable @var initializer(@scc) : !spirv.ptr, Uniform> + spirv.SpecConstant @sc0 = 1 : i8 + spirv.SpecConstant @sc1 = 2 : i8 + spirv.SpecConstant @sc2 = 3 : i8 + + spirv.SpecConstantComposite @scc (@sc0, @sc1, @sc2) : !spirv.array<3 x i8> + + spirv.GlobalVariable @var initializer(@scc) : !spirv.ptr, Uniform> +} + +// ----- + spirv.module Logical GLSL450 requires #spirv.vce { spirv.GlobalVariable @globalInvocationID built_in("GlobalInvocationId") : !spirv.ptr, Input> spirv.func @foo() "None" { -- GitLab From 4f215fdd62d3f014750339eab9a46946b6fb1c4a Mon Sep 17 00:00:00 2001 From: Konstantin Varlamov Date: Fri, 5 Jan 2024 16:29:23 -0800 Subject: [PATCH 169/728] [libc++][hardening] Categorize more assertions. (#75918) Also introduce `_LIBCPP_ASSERT_PEDANTIC` for assertions violating which results in a no-op or other benign behavior, but which may nevertheless indicate a bug in the invoking code. --- libcxx/include/__algorithm/pop_heap.h | 3 +- libcxx/include/__algorithm/sift_down.h | 2 +- libcxx/include/__algorithm/sort.h | 4 +- libcxx/include/__charconv/to_chars_base_10.h | 6 +- libcxx/include/__charconv/to_chars_integral.h | 2 +- libcxx/include/__charconv/traits.h | 4 +- .../include/__chrono/parser_std_format_spec.h | 7 +-- libcxx/include/__config | 7 +++ .../include/__filesystem/directory_iterator.h | 3 +- libcxx/include/__filesystem/path_iterator.h | 4 +- libcxx/include/__format/buffer.h | 12 ++-- libcxx/include/__format/format_arg.h | 2 +- libcxx/include/__format/formatter_bool.h | 2 +- .../__format/formatter_floating_point.h | 55 +++++++++---------- libcxx/include/__format/formatter_integral.h | 16 ++---- libcxx/include/__format/formatter_output.h | 6 +- libcxx/include/__format/formatter_string.h | 5 +- .../include/__format/parser_std_format_spec.h | 7 +-- libcxx/include/__format/range_formatter.h | 5 +- libcxx/include/__format/unicode.h | 14 ++--- libcxx/include/__format/write_escaped.h | 2 +- libcxx/include/__hash_table | 5 +- libcxx/include/__iterator/advance.h | 13 +++-- libcxx/include/__iterator/next.h | 6 +- libcxx/include/__iterator/prev.h | 6 +- .../__random/negative_binomial_distribution.h | 7 +-- libcxx/include/__ranges/chunk_by_view.h | 20 ++++--- libcxx/include/__ranges/drop_while_view.h | 3 +- libcxx/include/__ranges/filter_view.h | 3 +- libcxx/include/__thread/thread.h | 2 +- libcxx/include/__utility/exception_guard.h | 2 +- libcxx/include/__utility/unreachable.h | 2 +- libcxx/include/print | 8 ++- libcxx/include/regex | 23 +++++--- libcxx/include/set | 32 +++++------ libcxx/src/filesystem/error.h | 2 +- libcxx/src/filesystem/format_string.h | 2 +- libcxx/src/filesystem/posix_compat.h | 6 +- libcxx/src/include/to_chars_floating_point.h | 20 +++---- libcxx/src/memory_resource.cpp | 2 +- libcxx/src/strstream.cpp | 2 +- libcxx/src/system_error.cpp | 2 +- ...ert.exception_guard.no_exceptions.pass.cpp | 2 +- .../assert.unreachable.pass.cpp | 2 +- 44 files changed, 184 insertions(+), 156 deletions(-) diff --git a/libcxx/include/__algorithm/pop_heap.h b/libcxx/include/__algorithm/pop_heap.h index a93a9875f705..798a1d09934b 100644 --- a/libcxx/include/__algorithm/pop_heap.h +++ b/libcxx/include/__algorithm/pop_heap.h @@ -36,7 +36,8 @@ __pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare& __comp, typename iterator_traits<_RandomAccessIterator>::difference_type __len) { - _LIBCPP_ASSERT_UNCATEGORIZED(__len > 0, "The heap given to pop_heap must be non-empty"); + // Calling `pop_heap` on an empty range is undefined behavior, but in practice it will be a no-op. + _LIBCPP_ASSERT_PEDANTIC(__len > 0, "The heap given to pop_heap must be non-empty"); __comp_ref_type<_Compare> __comp_ref = __comp; diff --git a/libcxx/include/__algorithm/sift_down.h b/libcxx/include/__algorithm/sift_down.h index 7f152e4dbd7f..42803e30631f 100644 --- a/libcxx/include/__algorithm/sift_down.h +++ b/libcxx/include/__algorithm/sift_down.h @@ -85,7 +85,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator __floy _Compare&& __comp, typename iterator_traits<_RandomAccessIterator>::difference_type __len) { using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type; - _LIBCPP_ASSERT_UNCATEGORIZED(__len >= 2, "shouldn't be called unless __len >= 2"); + _LIBCPP_ASSERT_INTERNAL(__len >= 2, "shouldn't be called unless __len >= 2"); _RandomAccessIterator __hole = __first; _RandomAccessIterator __child_i = __first; diff --git a/libcxx/include/__algorithm/sort.h b/libcxx/include/__algorithm/sort.h index 1b878c33c7a1..ac47489af0aa 100644 --- a/libcxx/include/__algorithm/sort.h +++ b/libcxx/include/__algorithm/sort.h @@ -533,7 +533,7 @@ __bitset_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, using _Ops = _IterOps<_AlgPolicy>; typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type; - _LIBCPP_ASSERT_UNCATEGORIZED(__last - __first >= difference_type(3), ""); + _LIBCPP_ASSERT_INTERNAL(__last - __first >= difference_type(3), ""); const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around const _RandomAccessIterator __end = __last; (void)__end; // @@ -625,7 +625,7 @@ __partition_with_equals_on_right(_RandomAccessIterator __first, _RandomAccessIte using _Ops = _IterOps<_AlgPolicy>; typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; - _LIBCPP_ASSERT_UNCATEGORIZED(__last - __first >= difference_type(3), ""); + _LIBCPP_ASSERT_INTERNAL(__last - __first >= difference_type(3), ""); const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around const _RandomAccessIterator __end = __last; (void)__end; // diff --git a/libcxx/include/__charconv/to_chars_base_10.h b/libcxx/include/__charconv/to_chars_base_10.h index 33c512e20f04..0dee351521f9 100644 --- a/libcxx/include/__charconv/to_chars_base_10.h +++ b/libcxx/include/__charconv/to_chars_base_10.h @@ -132,14 +132,14 @@ __base_10_u64(char* __buffer, uint64_t __value) noexcept { /// range that can be used. However the range is sufficient for /// \ref __base_10_u128. _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline __uint128_t __pow_10(int __exp) noexcept { - _LIBCPP_ASSERT_UNCATEGORIZED(__exp >= __pow10_128_offset, "Index out of bounds"); + _LIBCPP_ASSERT_INTERNAL(__exp >= __pow10_128_offset, "Index out of bounds"); return __pow10_128[__exp - __pow10_128_offset]; } _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI inline char* __base_10_u128(char* __buffer, __uint128_t __value) noexcept { - _LIBCPP_ASSERT_UNCATEGORIZED( - __value > numeric_limits::max(), "The optimizations for this algorithm fail when this isn't true."); + _LIBCPP_ASSERT_INTERNAL( + __value > numeric_limits::max(), "The optimizations for this algorithm fails when this isn't true."); // Unlike the 64 to 32 bit case the 128 bit case the "upper half" can't be // stored in the "lower half". Instead we first need to handle the top most diff --git a/libcxx/include/__charconv/to_chars_integral.h b/libcxx/include/__charconv/to_chars_integral.h index f50cc55a4c6d..40fbe334d8d5 100644 --- a/libcxx/include/__charconv/to_chars_integral.h +++ b/libcxx/include/__charconv/to_chars_integral.h @@ -246,7 +246,7 @@ __to_chars_integral(char* __first, char* __last, _Tp __value) { template _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI int __to_chars_integral_width(_Tp __value, unsigned __base) { - _LIBCPP_ASSERT_UNCATEGORIZED(__value >= 0, "The function requires a non-negative value."); + _LIBCPP_ASSERT_INTERNAL(__value >= 0, "The function requires a non-negative value."); unsigned __base_2 = __base * __base; unsigned __base_3 = __base_2 * __base; diff --git a/libcxx/include/__charconv/traits.h b/libcxx/include/__charconv/traits.h index d3884b560dfd..b4907c3f7757 100644 --- a/libcxx/include/__charconv/traits.h +++ b/libcxx/include/__charconv/traits.h @@ -101,11 +101,11 @@ struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t numeric_limits::max(), "The optimizations for this algorithm fail when this isn't true."); // There's always a bit set in the upper 64-bits. auto __t = (128 - std::__libcpp_clz(static_cast(__v >> 64))) * 1233 >> 12; - _LIBCPP_ASSERT_UNCATEGORIZED(__t >= __itoa::__pow10_128_offset, "Index out of bounds"); + _LIBCPP_ASSERT_INTERNAL(__t >= __itoa::__pow10_128_offset, "Index out of bounds"); // __t is adjusted since the lookup table misses the lower entries. return __t - (__v < __itoa::__pow10_128[__t - __itoa::__pow10_128_offset]) + 1; } diff --git a/libcxx/include/__chrono/parser_std_format_spec.h b/libcxx/include/__chrono/parser_std_format_spec.h index 296be8794ec5..785bbae198e4 100644 --- a/libcxx/include/__chrono/parser_std_format_spec.h +++ b/libcxx/include/__chrono/parser_std_format_spec.h @@ -160,10 +160,9 @@ public: private: _LIBCPP_HIDE_FROM_ABI constexpr _ConstIterator __parse_chrono_specs(_ConstIterator __begin, _ConstIterator __end, __flags __flags) { - _LIBCPP_ASSERT_UNCATEGORIZED( - __begin != __end, - "When called with an empty input the function will cause " - "undefined behavior by evaluating data not in the input"); + _LIBCPP_ASSERT_INTERNAL(__begin != __end, + "When called with an empty input the function will cause " + "undefined behavior by evaluating data not in the input"); if (*__begin != _CharT('%') && *__begin != _CharT('}')) std::__throw_format_error("The format specifier expects a '%' or a '}'"); diff --git a/libcxx/include/__config b/libcxx/include/__config index 40e6da8bc03a..082c73e672c7 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -283,6 +283,9 @@ // - `_LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR` -- checks any operations that exchange nodes between containers to make sure // the containers have compatible allocators. // +// - `_LIBCPP_ASSERT_PEDANTIC` -- checks prerequisites which are imposed by the Standard, but violating which happens to +// be benign in our implementation. +// // - `_LIBCPP_ASSERT_INTERNAL` -- checks that internal invariants of the library hold. These assertions don't depend on // user input. // @@ -325,6 +328,7 @@ _LIBCPP_HARDENING_MODE_DEBUG // vulnerability. # define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message) _LIBCPP_ASSUME(expression) # define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message) _LIBCPP_ASSUME(expression) +# define _LIBCPP_ASSERT_PEDANTIC(expression, message) _LIBCPP_ASSUME(expression) # define _LIBCPP_ASSERT_INTERNAL(expression, message) _LIBCPP_ASSUME(expression) # define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message) _LIBCPP_ASSUME(expression) @@ -339,6 +343,7 @@ _LIBCPP_HARDENING_MODE_DEBUG # define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message) _LIBCPP_ASSERT(expression, message) # define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message) _LIBCPP_ASSERT(expression, message) # define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message) _LIBCPP_ASSERT(expression, message) +# define _LIBCPP_ASSERT_PEDANTIC(expression, message) _LIBCPP_ASSERT(expression, message) // Disabled checks. # define _LIBCPP_ASSERT_INTERNAL(expression, message) _LIBCPP_ASSUME(expression) @@ -352,6 +357,7 @@ _LIBCPP_HARDENING_MODE_DEBUG # define _LIBCPP_ASSERT_NON_NULL(expression, message) _LIBCPP_ASSERT(expression, message) # define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message) _LIBCPP_ASSERT(expression, message) # define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message) _LIBCPP_ASSERT(expression, message) +# define _LIBCPP_ASSERT_PEDANTIC(expression, message) _LIBCPP_ASSERT(expression, message) # define _LIBCPP_ASSERT_INTERNAL(expression, message) _LIBCPP_ASSERT(expression, message) # define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message) _LIBCPP_ASSERT(expression, message) @@ -365,6 +371,7 @@ _LIBCPP_HARDENING_MODE_DEBUG # define _LIBCPP_ASSERT_NON_NULL(expression, message) _LIBCPP_ASSUME(expression) # define _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(expression, message) _LIBCPP_ASSUME(expression) # define _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(expression, message) _LIBCPP_ASSUME(expression) +# define _LIBCPP_ASSERT_PEDANTIC(expression, message) _LIBCPP_ASSUME(expression) # define _LIBCPP_ASSERT_INTERNAL(expression, message) _LIBCPP_ASSUME(expression) # define _LIBCPP_ASSERT_UNCATEGORIZED(expression, message) _LIBCPP_ASSUME(expression) diff --git a/libcxx/include/__filesystem/directory_iterator.h b/libcxx/include/__filesystem/directory_iterator.h index 29bd8da6caa4..5287a4d8b055 100644 --- a/libcxx/include/__filesystem/directory_iterator.h +++ b/libcxx/include/__filesystem/directory_iterator.h @@ -73,7 +73,8 @@ public: _LIBCPP_HIDE_FROM_ABI ~directory_iterator() = default; _LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__imp_, "The end iterator cannot be dereferenced"); + // Note: this check duplicates a check in `__dereference()`. + _LIBCPP_ASSERT_NON_NULL(__imp_, "The end iterator cannot be dereferenced"); return __dereference(); } diff --git a/libcxx/include/__filesystem/path_iterator.h b/libcxx/include/__filesystem/path_iterator.h index 1a9aaf0e7d99..d2d65cd122ca 100644 --- a/libcxx/include/__filesystem/path_iterator.h +++ b/libcxx/include/__filesystem/path_iterator.h @@ -61,7 +61,7 @@ public: _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return &__stashed_elem_; } _LIBCPP_HIDE_FROM_ABI iterator& operator++() { - _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _Singular, "attempting to increment a singular iterator"); + _LIBCPP_ASSERT_NON_NULL(__state_ != _Singular, "attempting to increment a singular iterator"); _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _AtEnd, "attempting to increment the end iterator"); return __increment(); } @@ -73,7 +73,7 @@ public: } _LIBCPP_HIDE_FROM_ABI iterator& operator--() { - _LIBCPP_ASSERT_UNCATEGORIZED(__state_ != _Singular, "attempting to decrement a singular iterator"); + _LIBCPP_ASSERT_NON_NULL(__state_ != _Singular, "attempting to decrement a singular iterator"); _LIBCPP_ASSERT_UNCATEGORIZED( __entry_.data() != __path_ptr_->native().data(), "attempting to decrement the begin iterator"); return __decrement(); diff --git a/libcxx/include/__format/buffer.h b/libcxx/include/__format/buffer.h index 7ee583d81394..8598f0a1c039 100644 --- a/libcxx/include/__format/buffer.h +++ b/libcxx/include/__format/buffer.h @@ -115,7 +115,7 @@ public: // The output doesn't fit in the internal buffer. // Copy the data in "__capacity_" sized chunks. - _LIBCPP_ASSERT_UNCATEGORIZED(__size_ == 0, "the buffer should be flushed by __flush_on_overflow"); + _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow"); const _InCharT* __first = __str.data(); do { size_t __chunk = std::min(__n, __capacity_); @@ -134,7 +134,7 @@ public: class _UnaryOperation, __fmt_char_type _InCharT = typename iterator_traits<_Iterator>::value_type> _LIBCPP_HIDE_FROM_ABI void __transform(_Iterator __first, _Iterator __last, _UnaryOperation __operation) { - _LIBCPP_ASSERT_UNCATEGORIZED(__first <= __last, "not a valid range"); + _LIBCPP_ASSERT_INTERNAL(__first <= __last, "not a valid range"); size_t __n = static_cast(__last - __first); __flush_on_overflow(__n); @@ -146,7 +146,7 @@ public: // The output doesn't fit in the internal buffer. // Transform the data in "__capacity_" sized chunks. - _LIBCPP_ASSERT_UNCATEGORIZED(__size_ == 0, "the buffer should be flushed by __flush_on_overflow"); + _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow"); do { size_t __chunk = std::min(__n, __capacity_); std::transform(__first, __first + __chunk, std::addressof(__ptr_[__size_]), __operation); @@ -168,7 +168,7 @@ public: // The output doesn't fit in the internal buffer. // Fill the buffer in "__capacity_" sized chunks. - _LIBCPP_ASSERT_UNCATEGORIZED(__size_ == 0, "the buffer should be flushed by __flush_on_overflow"); + _LIBCPP_ASSERT_INTERNAL(__size_ == 0, "the buffer should be flushed by __flush_on_overflow"); do { size_t __chunk = std::min(__n, __capacity_); std::fill_n(std::addressof(__ptr_[__size_]), __chunk, __value); @@ -596,7 +596,7 @@ public: class _UnaryOperation, __fmt_char_type _InCharT = typename iterator_traits<_Iterator>::value_type> _LIBCPP_HIDE_FROM_ABI void __transform(_Iterator __first, _Iterator __last, _UnaryOperation __operation) { - _LIBCPP_ASSERT_UNCATEGORIZED(__first <= __last, "not a valid range"); + _LIBCPP_ASSERT_INTERNAL(__first <= __last, "not a valid range"); size_t __n = static_cast(__last - __first); if (__size_ + __n >= __capacity_) @@ -623,7 +623,7 @@ private: _LIBCPP_HIDE_FROM_ABI void __grow_buffer() { __grow_buffer(__capacity_ * 1.6); } _LIBCPP_HIDE_FROM_ABI void __grow_buffer(size_t __capacity) { - _LIBCPP_ASSERT_UNCATEGORIZED(__capacity > __capacity_, "the buffer must grow"); + _LIBCPP_ASSERT_INTERNAL(__capacity > __capacity_, "the buffer must grow"); auto __result = std::__allocate_at_least(__alloc_, __capacity); auto __guard = std::__make_exception_guard([&] { allocator_traits<_Alloc>::deallocate(__alloc_, __result.ptr, __result.count); diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h index 280c91082417..10fca15d5a7a 100644 --- a/libcxx/include/__format/format_arg.h +++ b/libcxx/include/__format/format_arg.h @@ -83,7 +83,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __use_packed_format_arg_store(size_t __size } _LIBCPP_HIDE_FROM_ABI constexpr __arg_t __get_packed_type(uint64_t __types, size_t __id) { - _LIBCPP_ASSERT_UNCATEGORIZED(__id <= __packed_types_max, ""); + _LIBCPP_ASSERT_INTERNAL(__id <= __packed_types_max, ""); if (__id > 0) __types >>= __id * __packed_arg_t_bits; diff --git a/libcxx/include/__format/formatter_bool.h b/libcxx/include/__format/formatter_bool.h index 3c8ae95f55fa..1c479501b675 100644 --- a/libcxx/include/__format/formatter_bool.h +++ b/libcxx/include/__format/formatter_bool.h @@ -62,7 +62,7 @@ public: static_cast(__value), __ctx, __parser_.__get_parsed_std_specifications(__ctx)); default: - _LIBCPP_ASSERT_UNCATEGORIZED(false, "The parse function should have validated the type"); + _LIBCPP_ASSERT_INTERNAL(false, "The parse function should have validated the type"); __libcpp_unreachable(); } } diff --git a/libcxx/include/__format/formatter_floating_point.h b/libcxx/include/__format/formatter_floating_point.h index 33cc2a4ed661..6802a8b7bd4c 100644 --- a/libcxx/include/__format/formatter_floating_point.h +++ b/libcxx/include/__format/formatter_floating_point.h @@ -57,21 +57,21 @@ namespace __formatter { template _LIBCPP_HIDE_FROM_ABI char* __to_buffer(char* __first, char* __last, _Tp __value) { to_chars_result __r = std::to_chars(__first, __last, __value); - _LIBCPP_ASSERT_UNCATEGORIZED(__r.ec == errc(0), "Internal buffer too small"); + _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small"); return __r.ptr; } template _LIBCPP_HIDE_FROM_ABI char* __to_buffer(char* __first, char* __last, _Tp __value, chars_format __fmt) { to_chars_result __r = std::to_chars(__first, __last, __value, __fmt); - _LIBCPP_ASSERT_UNCATEGORIZED(__r.ec == errc(0), "Internal buffer too small"); + _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small"); return __r.ptr; } template _LIBCPP_HIDE_FROM_ABI char* __to_buffer(char* __first, char* __last, _Tp __value, chars_format __fmt, int __precision) { to_chars_result __r = std::to_chars(__first, __last, __value, __fmt, __precision); - _LIBCPP_ASSERT_UNCATEGORIZED(__r.ec == errc(0), "Internal buffer too small"); + _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small"); return __r.ptr; } @@ -252,10 +252,10 @@ __format_buffer_default(const __float_buffer<_Fp>& __buffer, _Tp __value, char* __result.__radix_point = __result.__last; // clang-format off - _LIBCPP_ASSERT_UNCATEGORIZED((__result.__integral != __result.__last) && - (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && - (__result.__exponent == __result.__last || *__result.__exponent == 'e'), - "Post-condition failure."); + _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) && + (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && + (__result.__exponent == __result.__last || *__result.__exponent == 'e'), + "Post-condition failure."); // clang-format on return __result; @@ -304,10 +304,10 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_hexadecimal_lower_case( } // clang-format off - _LIBCPP_ASSERT_UNCATEGORIZED((__result.__integral != __result.__last) && - (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && - (__result.__exponent != __result.__last && *__result.__exponent == 'p'), - "Post-condition failure."); + _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) && + (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && + (__result.__exponent != __result.__last && *__result.__exponent == 'p'), + "Post-condition failure."); // clang-format on return __result; @@ -332,7 +332,7 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_lower_case( __formatter::__to_buffer(__integral, __buffer.end(), __value, chars_format::scientific, __precision); char* __first = __integral + 1; - _LIBCPP_ASSERT_UNCATEGORIZED(__first != __result.__last, "No exponent present"); + _LIBCPP_ASSERT_INTERNAL(__first != __result.__last, "No exponent present"); if (*__first == '.') { __result.__radix_point = __first; __result.__exponent = __formatter::__find_exponent(__first + 1, __result.__last); @@ -342,10 +342,10 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer_scientific_lower_case( } // clang-format off - _LIBCPP_ASSERT_UNCATEGORIZED((__result.__integral != __result.__last) && - (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && - (__result.__exponent != __result.__last && *__result.__exponent == 'e'), - "Post-condition failure."); + _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) && + (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && + (__result.__exponent != __result.__last && *__result.__exponent == 'e'), + "Post-condition failure."); // clang-format on return __result; } @@ -374,10 +374,10 @@ __format_buffer_fixed(const __float_buffer<_Fp>& __buffer, _Tp __value, int __pr __result.__exponent = __result.__last; // clang-format off - _LIBCPP_ASSERT_UNCATEGORIZED((__result.__integral != __result.__last) && - (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && - (__result.__exponent == __result.__last), - "Post-condition failure."); + _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) && + (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && + (__result.__exponent == __result.__last), + "Post-condition failure."); // clang-format on return __result; } @@ -410,10 +410,10 @@ __format_buffer_general_lower_case(__float_buffer<_Fp>& __buffer, _Tp __value, i } // clang-format off - _LIBCPP_ASSERT_UNCATEGORIZED((__result.__integral != __result.__last) && - (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && - (__result.__exponent == __result.__last || *__result.__exponent == 'e'), - "Post-condition failure."); + _LIBCPP_ASSERT_INTERNAL((__result.__integral != __result.__last) && + (__result.__radix_point == __result.__last || *__result.__radix_point == '.') && + (__result.__exponent == __result.__last || *__result.__exponent == 'e'), + "Post-condition failure."); // clang-format on return __result; @@ -485,7 +485,7 @@ _LIBCPP_HIDE_FROM_ABI __float_result __format_buffer( return __formatter::__format_buffer_general_upper_case(__buffer, __value, __buffer.__precision(), __first); default: - _LIBCPP_ASSERT_UNCATEGORIZED(false, "The parser should have validated the type"); + _LIBCPP_ASSERT_INTERNAL(false, "The parser should have validated the type"); __libcpp_unreachable(); } } @@ -620,9 +620,8 @@ _LIBCPP_HIDE_FROM_ABI auto __write_using_trailing_zeros( size_t __size, const _CharT* __exponent, size_t __num_trailing_zeros) -> decltype(__out_it) { - _LIBCPP_ASSERT_UNCATEGORIZED(__first <= __last, "Not a valid range"); - _LIBCPP_ASSERT_UNCATEGORIZED( - __num_trailing_zeros > 0, "The overload not writing trailing zeros should have been used"); + _LIBCPP_ASSERT_INTERNAL(__first <= __last, "Not a valid range"); + _LIBCPP_ASSERT_INTERNAL(__num_trailing_zeros > 0, "The overload not writing trailing zeros should have been used"); __padding_size_result __padding = __formatter::__padding_size(__size + __num_trailing_zeros, __specs.__width_, __specs.__alignment_); diff --git a/libcxx/include/__format/formatter_integral.h b/libcxx/include/__format/formatter_integral.h index ca66e26ede10..e0217a240027 100644 --- a/libcxx/include/__format/formatter_integral.h +++ b/libcxx/include/__format/formatter_integral.h @@ -90,10 +90,8 @@ _LIBCPP_HIDE_FROM_ABI inline _Iterator __insert_sign(_Iterator __buf, bool __neg * regardless whether the @c std::numpunct's type is @c char or @c wchar_t. */ _LIBCPP_HIDE_FROM_ABI inline string __determine_grouping(ptrdiff_t __size, const string& __grouping) { - _LIBCPP_ASSERT_UNCATEGORIZED( - !__grouping.empty() && __size > __grouping[0], - "The slow grouping formatting is used while there will be no " - "separators written"); + _LIBCPP_ASSERT_INTERNAL(!__grouping.empty() && __size > __grouping[0], + "The slow grouping formatting is used while there will be no separators written"); string __r; auto __end = __grouping.end() - 1; auto __ptr = __grouping.begin(); @@ -161,7 +159,7 @@ _LIBCPP_HIDE_FROM_ABI _Iterator __to_buffer(_Iterator __first, _Iterator __last, // TODO FMT Evaluate code overhead due to not calling the internal function // directly. (Should be zero overhead.) to_chars_result __r = std::to_chars(std::to_address(__first), std::to_address(__last), __value, __base); - _LIBCPP_ASSERT_UNCATEGORIZED(__r.ec == errc(0), "Internal buffer too small"); + _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small"); auto __diff = __r.ptr - std::to_address(__first); return __first + __diff; } @@ -248,10 +246,8 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __write_using_decimal_separators( auto __r = __grouping.rbegin(); auto __e = __grouping.rend() - 1; - _LIBCPP_ASSERT_UNCATEGORIZED( - __r != __e, - "The slow grouping formatting is used while " - "there will be no separators written."); + _LIBCPP_ASSERT_INTERNAL( + __r != __e, "The slow grouping formatting is used while there will be no separators written."); // The output is divided in small groups of numbers to write: // - A group before the first separator. // - A separator and a group, repeated for the number of separators. @@ -380,7 +376,7 @@ __format_integer(_Tp __value, return __formatter::__format_integer(__value, __ctx, __specs, __negative, __array.begin(), __array.end(), "0X", 16); } default: - _LIBCPP_ASSERT_UNCATEGORIZED(false, "The parse function should have validated the type"); + _LIBCPP_ASSERT_INTERNAL(false, "The parse function should have validated the type"); __libcpp_unreachable(); } } diff --git a/libcxx/include/__format/formatter_output.h b/libcxx/include/__format/formatter_output.h index 31e06425703a..eebe880d69ef 100644 --- a/libcxx/include/__format/formatter_output.h +++ b/libcxx/include/__format/formatter_output.h @@ -66,8 +66,8 @@ struct _LIBCPP_EXPORTED_FROM_ABI __padding_size_result { _LIBCPP_HIDE_FROM_ABI constexpr __padding_size_result __padding_size(size_t __size, size_t __width, __format_spec::__alignment __align) { - _LIBCPP_ASSERT_UNCATEGORIZED(__width > __size, "don't call this function when no padding is required"); - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_INTERNAL(__width > __size, "don't call this function when no padding is required"); + _LIBCPP_ASSERT_INTERNAL( __align != __format_spec::__alignment::__zero_padding, "the caller should have handled the zero-padding"); size_t __fill = __width - __size; @@ -296,7 +296,7 @@ _LIBCPP_HIDE_FROM_ABI auto __write_string_no_precision( basic_string_view<_CharT> __str, output_iterator auto __out_it, __format_spec::__parsed_specifications<_CharT> __specs) -> decltype(__out_it) { - _LIBCPP_ASSERT_UNCATEGORIZED(!__specs.__has_precision(), "use __write_string"); + _LIBCPP_ASSERT_INTERNAL(!__specs.__has_precision(), "use __write_string"); // No padding -> copy the string if (!__specs.__has_width()) diff --git a/libcxx/include/__format/formatter_string.h b/libcxx/include/__format/formatter_string.h index 4ba5617a49c8..d1ccfb9b5f7d 100644 --- a/libcxx/include/__format/formatter_string.h +++ b/libcxx/include/__format/formatter_string.h @@ -64,10 +64,7 @@ struct _LIBCPP_TEMPLATE_VIS formatter : public __formatte template _LIBCPP_HIDE_FROM_ABI typename _FormatContext::iterator format(const _CharT* __str, _FormatContext& __ctx) const { - _LIBCPP_ASSERT_UNCATEGORIZED( - __str, - "The basic_format_arg constructor should have " - "prevented an invalid pointer."); + _LIBCPP_ASSERT_INTERNAL(__str, "The basic_format_arg constructor should have prevented an invalid pointer."); __format_spec::__parsed_specifications<_CharT> __specs = _Base::__parser_.__get_parsed_std_specifications(__ctx); # if _LIBCPP_STD_VER >= 23 diff --git a/libcxx/include/__format/parser_std_format_spec.h b/libcxx/include/__format/parser_std_format_spec.h index e38729db965c..cf8af87b2128 100644 --- a/libcxx/include/__format/parser_std_format_spec.h +++ b/libcxx/include/__format/parser_std_format_spec.h @@ -733,10 +733,9 @@ private: __format::__parse_number_result __r = __format::__parse_number(__begin, __end); __width_ = __r.__value; - _LIBCPP_ASSERT_UNCATEGORIZED( - __width_ != 0, - "A zero value isn't allowed and should be impossible, " - "due to validations in this function"); + _LIBCPP_ASSERT_INTERNAL(__width_ != 0, + "A zero value isn't allowed and should be impossible, " + "due to validations in this function"); __begin = __r.__last; return true; } diff --git a/libcxx/include/__format/range_formatter.h b/libcxx/include/__format/range_formatter.h index d13278009fcf..691563074349 100644 --- a/libcxx/include/__format/range_formatter.h +++ b/libcxx/include/__format/range_formatter.h @@ -246,9 +246,8 @@ private: __parse_empty_range_underlying_spec(_ParseContext& __ctx, typename _ParseContext::iterator __begin) { __ctx.advance_to(__begin); [[maybe_unused]] typename _ParseContext::iterator __result = __underlying_.parse(__ctx); - _LIBCPP_ASSERT_UNCATEGORIZED( - __result == __begin, - "the underlying's parse function should not advance the input beyond the end of the input"); + _LIBCPP_ASSERT_INTERNAL(__result == __begin, + "the underlying's parse function should not advance the input beyond the end of the input"); return __begin; } diff --git a/libcxx/include/__format/unicode.h b/libcxx/include/__format/unicode.h index 8e1e7bb192a0..40067ca3448b 100644 --- a/libcxx/include/__format/unicode.h +++ b/libcxx/include/__format/unicode.h @@ -153,7 +153,7 @@ public: // - The parser always needs to consume these code units // - The code is optimized for well-formed UTF-8 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { - _LIBCPP_ASSERT_UNCATEGORIZED(__first_ != __last_, "can't move beyond the end of input"); + _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); // Based on the number of leading 1 bits the number of code units in the // code point can be determined. See @@ -259,7 +259,7 @@ public: _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; } [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { - _LIBCPP_ASSERT_UNCATEGORIZED(__first_ != __last_, "can't move beyond the end of input"); + _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); char32_t __value = static_cast(*__first_++); if constexpr (sizeof(wchar_t) == 2) { @@ -305,8 +305,8 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __at_extended_grapheme_cluster_break( // *** Break at the start and end of text, unless the text is empty. *** - _LIBCPP_ASSERT_UNCATEGORIZED(__prev != __property::__sot, "should be handled in the constructor"); // GB1 - _LIBCPP_ASSERT_UNCATEGORIZED(__prev != __property::__eot, "should be handled by our caller"); // GB2 + _LIBCPP_ASSERT_INTERNAL(__prev != __property::__sot, "should be handled in the constructor"); // GB1 + _LIBCPP_ASSERT_INTERNAL(__prev != __property::__eot, "should be handled by our caller"); // GB2 // *** Do not break between a CR and LF. Otherwise, break before and after controls. *** if (__prev == __property::__CR && __next == __property::__LF) // GB3 @@ -401,8 +401,8 @@ public: }; _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() { - _LIBCPP_ASSERT_UNCATEGORIZED(__next_prop_ != __extended_grapheme_custer_property_boundary::__property::__eot, - "can't move beyond the end of input"); + _LIBCPP_ASSERT_INTERNAL(__next_prop_ != __extended_grapheme_custer_property_boundary::__property::__eot, + "can't move beyond the end of input"); char32_t __code_point = __next_code_point_; if (!__code_point_view_.__at_end()) @@ -459,7 +459,7 @@ public: _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; } [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { - _LIBCPP_ASSERT_UNCATEGORIZED(__first_ != __last_, "can't move beyond the end of input"); + _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); return {static_cast(*__first_++)}; } diff --git a/libcxx/include/__format/write_escaped.h b/libcxx/include/__format/write_escaped.h index 15141eebc029..ec1283a173e9 100644 --- a/libcxx/include/__format/write_escaped.h +++ b/libcxx/include/__format/write_escaped.h @@ -71,7 +71,7 @@ __write_escaped_code_unit(basic_string<_CharT>& __str, char32_t __value, const _ char __buffer[8]; to_chars_result __r = std::to_chars(std::begin(__buffer), std::end(__buffer), __value, 16); - _LIBCPP_ASSERT_UNCATEGORIZED(__r.ec == errc(0), "Internal buffer too small"); + _LIBCPP_ASSERT_INTERNAL(__r.ec == errc(0), "Internal buffer too small"); std::ranges::copy(std::begin(__buffer), __r.ptr, __out_it); __str += _CharT('}'); diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table index 3cee48ef8538..4ca49fe42606 100644 --- a/libcxx/include/__hash_table +++ b/libcxx/include/__hash_table @@ -915,7 +915,10 @@ public: return __bc != 0 ? (float)size() / __bc : 0.f; } _LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) _NOEXCEPT { - _LIBCPP_ASSERT_UNCATEGORIZED(__mlf > 0, "unordered container::max_load_factor(lf) called with lf <= 0"); + // While passing a non-positive load factor is undefined behavior, in practice the result will be benign (the + // call will be equivalent to `max_load_factor(load_factor())`, which is also the case for passing a valid value + // less than the current `load_factor`). + _LIBCPP_ASSERT_PEDANTIC(__mlf > 0, "unordered container::max_load_factor(lf) called with lf <= 0"); max_load_factor() = std::max(__mlf, load_factor()); } diff --git a/libcxx/include/__iterator/advance.h b/libcxx/include/__iterator/advance.h index 64c8d249f78f..73473f899eac 100644 --- a/libcxx/include/__iterator/advance.h +++ b/libcxx/include/__iterator/advance.h @@ -65,8 +65,9 @@ template < class _InputIter, _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 void advance(_InputIter& __i, _Distance __orig_n) { typedef typename iterator_traits<_InputIter>::difference_type _Difference; _Difference __n = static_cast<_Difference>(std::__convert_to_integral(__orig_n)); - _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value, - "Attempt to advance(it, n) with negative n on a non-bidirectional iterator"); + // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation. + _LIBCPP_ASSERT_PEDANTIC(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value, + "Attempt to advance(it, n) with negative n on a non-bidirectional iterator"); std::__advance(__i, __n, typename iterator_traits<_InputIter>::iterator_category()); } @@ -99,7 +100,8 @@ public: // Preconditions: If `I` does not model `bidirectional_iterator`, `n` is not negative. template _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Ip& __i, iter_difference_t<_Ip> __n) const { - _LIBCPP_ASSERT_UNCATEGORIZED( + // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation. + _LIBCPP_ASSERT_PEDANTIC( __n >= 0 || bidirectional_iterator<_Ip>, "If `n < 0`, then `bidirectional_iterator` must be true."); // If `I` models `random_access_iterator`, equivalent to `i += n`. @@ -149,8 +151,9 @@ public: template _Sp> _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip& __i, iter_difference_t<_Ip> __n, _Sp __bound_sentinel) const { - _LIBCPP_ASSERT_UNCATEGORIZED((__n >= 0) || (bidirectional_iterator<_Ip> && same_as<_Ip, _Sp>), - "If `n < 0`, then `bidirectional_iterator && same_as` must be true."); + // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation. + _LIBCPP_ASSERT_PEDANTIC((__n >= 0) || (bidirectional_iterator<_Ip> && same_as<_Ip, _Sp>), + "If `n < 0`, then `bidirectional_iterator && same_as` must be true."); // If `S` and `I` model `sized_sentinel_for`: if constexpr (sized_sentinel_for<_Sp, _Ip>) { // If |n| >= |bound_sentinel - i|, equivalent to `ranges::advance(i, bound_sentinel)`. diff --git a/libcxx/include/__iterator/next.h b/libcxx/include/__iterator/next.h index da60aacfd08d..21d3688ad9eb 100644 --- a/libcxx/include/__iterator/next.h +++ b/libcxx/include/__iterator/next.h @@ -27,8 +27,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD template ::value, int> = 0> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter next(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) { - _LIBCPP_ASSERT_UNCATEGORIZED(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value, - "Attempt to next(it, n) with negative n on a non-bidirectional iterator"); + // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation. + // Note that this check duplicates the similar check in `std::advance`. + _LIBCPP_ASSERT_PEDANTIC(__n >= 0 || __has_bidirectional_iterator_category<_InputIter>::value, + "Attempt to next(it, n) with negative n on a non-bidirectional iterator"); std::advance(__x, __n); return __x; diff --git a/libcxx/include/__iterator/prev.h b/libcxx/include/__iterator/prev.h index 1651942acea9..2f0e6a088edb 100644 --- a/libcxx/include/__iterator/prev.h +++ b/libcxx/include/__iterator/prev.h @@ -27,8 +27,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD template ::value, int> = 0> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _InputIter prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) { - _LIBCPP_ASSERT_UNCATEGORIZED(__n <= 0 || __has_bidirectional_iterator_category<_InputIter>::value, - "Attempt to prev(it, n) with a positive n on a non-bidirectional iterator"); + // Calling `advance` with a negative value on a non-bidirectional iterator is a no-op in the current implementation. + // Note that this check duplicates the similar check in `std::advance`. + _LIBCPP_ASSERT_PEDANTIC(__n <= 0 || __has_bidirectional_iterator_category<_InputIter>::value, + "Attempt to prev(it, n) with a positive n on a non-bidirectional iterator"); std::advance(__x, -__n); return __x; } diff --git a/libcxx/include/__random/negative_binomial_distribution.h b/libcxx/include/__random/negative_binomial_distribution.h index 580c74d46440..eed4f511e871 100644 --- a/libcxx/include/__random/negative_binomial_distribution.h +++ b/libcxx/include/__random/negative_binomial_distribution.h @@ -113,10 +113,9 @@ _IntType negative_binomial_distribution<_IntType>::operator()(_URNG& __urng, con else ++__f; } - _LIBCPP_ASSERT_UNCATEGORIZED( - __f >= 0, - "std::negative_binomial_distribution should never produce negative values. " - "This is almost certainly a signed integer overflow issue on __f."); + _LIBCPP_ASSERT_INTERNAL(__f >= 0, + "std::negative_binomial_distribution should never produce negative values. " + "This is almost certainly a signed integer overflow issue on __f."); return __f; } return poisson_distribution(gamma_distribution(__k, (1 - __p) / __p)(__urng))(__urng); diff --git a/libcxx/include/__ranges/chunk_by_view.h b/libcxx/include/__ranges/chunk_by_view.h index 3ecc018cac9d..c5b3240a7d0b 100644 --- a/libcxx/include/__ranges/chunk_by_view.h +++ b/libcxx/include/__ranges/chunk_by_view.h @@ -66,7 +66,8 @@ class _LIBCPP_ABI_2023_OVERLAPPING_SUBOBJECT_FIX_TAG chunk_by_view class __iterator; _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> __find_next(iterator_t<_View> __current) { - _LIBCPP_ASSERT_UNCATEGORIZED( + // Note: this duplicates a check in `optional` but provides a better error message. + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( __pred_.__has_value(), "Trying to call __find_next() on a chunk_by_view that does not have a valid predicate."); auto __reversed_pred = [this](_Tp&& __x, _Up&& __y) -> bool { return !std::invoke(*__pred_, std::forward<_Tp>(__x), std::forward<_Up>(__y)); @@ -78,9 +79,10 @@ class _LIBCPP_ABI_2023_OVERLAPPING_SUBOBJECT_FIX_TAG chunk_by_view _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> __find_prev(iterator_t<_View> __current) requires bidirectional_range<_View> { - _LIBCPP_ASSERT_UNCATEGORIZED( - __current != ranges::begin(__base_), "Trying to call __find_prev() on a begin iterator."); - _LIBCPP_ASSERT_UNCATEGORIZED( + // Attempting to decrement a begin iterator is a no-op (`__find_prev` would return the same argument given to it). + _LIBCPP_ASSERT_PEDANTIC(__current != ranges::begin(__base_), "Trying to call __find_prev() on a begin iterator."); + // Note: this duplicates a check in `optional` but provides a better error message. + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( __pred_.__has_value(), "Trying to call __find_prev() on a chunk_by_view that does not have a valid predicate."); auto __first = ranges::begin(__base_); @@ -110,7 +112,8 @@ public: _LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; } _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() { - _LIBCPP_ASSERT_UNCATEGORIZED( + // Note: this duplicates a check in `optional` but provides a better error message. + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( __pred_.__has_value(), "Trying to call begin() on a chunk_by_view that does not have a valid predicate."); auto __first = ranges::begin(__base_); @@ -154,12 +157,15 @@ public: _LIBCPP_HIDE_FROM_ABI __iterator() = default; _LIBCPP_HIDE_FROM_ABI constexpr value_type operator*() const { - _LIBCPP_ASSERT_UNCATEGORIZED(__current_ != __next_, "Trying to dereference past-the-end chunk_by_view iterator."); + // If the iterator is at end, this would return an empty range which can be checked by the calling code and doesn't + // necessarily lead to a bad access. + _LIBCPP_ASSERT_PEDANTIC(__current_ != __next_, "Trying to dereference past-the-end chunk_by_view iterator."); return {__current_, __next_}; } _LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() { - _LIBCPP_ASSERT_UNCATEGORIZED(__current_ != __next_, "Trying to increment past end chunk_by_view iterator."); + // Attempting to increment an end iterator is a no-op (`__find_next` would return the same argument given to it). + _LIBCPP_ASSERT_PEDANTIC(__current_ != __next_, "Trying to increment past end chunk_by_view iterator."); __current_ = __next_; __next_ = __parent_->__find_next(__current_); return *this; diff --git a/libcxx/include/__ranges/drop_while_view.h b/libcxx/include/__ranges/drop_while_view.h index eb3783eb42f1..b367f735c141 100644 --- a/libcxx/include/__ranges/drop_while_view.h +++ b/libcxx/include/__ranges/drop_while_view.h @@ -66,7 +66,8 @@ public: _LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; } _LIBCPP_HIDE_FROM_ABI constexpr auto begin() { - _LIBCPP_ASSERT_UNCATEGORIZED( + // Note: this duplicates a check in `optional` but provides a better error message. + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( __pred_.__has_value(), "drop_while_view needs to have a non-empty predicate before calling begin() -- did a previous " "assignment to this drop_while_view fail?"); diff --git a/libcxx/include/__ranges/filter_view.h b/libcxx/include/__ranges/filter_view.h index 868ad128e894..ecb78eee3810 100644 --- a/libcxx/include/__ranges/filter_view.h +++ b/libcxx/include/__ranges/filter_view.h @@ -83,7 +83,8 @@ public: _LIBCPP_HIDE_FROM_ABI constexpr _Pred const& pred() const { return *__pred_; } _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() { - _LIBCPP_ASSERT_UNCATEGORIZED( + // Note: this duplicates a check in `optional` but provides a better error message. + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS( __pred_.__has_value(), "Trying to call begin() on a filter_view that does not have a valid predicate."); if constexpr (_UseCache) { if (!__cached_begin_.__has_value()) { diff --git a/libcxx/include/__thread/thread.h b/libcxx/include/__thread/thread.h index f3300752ac9e..463bbd677255 100644 --- a/libcxx/include/__thread/thread.h +++ b/libcxx/include/__thread/thread.h @@ -104,7 +104,7 @@ __thread_specific_ptr<_Tp>::~__thread_specific_ptr() { template void __thread_specific_ptr<_Tp>::set_pointer(pointer __p) { - _LIBCPP_ASSERT_UNCATEGORIZED(get() == nullptr, "Attempting to overwrite thread local data"); + _LIBCPP_ASSERT_INTERNAL(get() == nullptr, "Attempting to overwrite thread local data"); std::__libcpp_tls_set(__key_, __p); } diff --git a/libcxx/include/__utility/exception_guard.h b/libcxx/include/__utility/exception_guard.h index 389fca6c7101..8d90dfd5f190 100644 --- a/libcxx/include/__utility/exception_guard.h +++ b/libcxx/include/__utility/exception_guard.h @@ -115,7 +115,7 @@ struct __exception_guard_noexceptions { } _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NODEBUG ~__exception_guard_noexceptions() { - _LIBCPP_ASSERT_UNCATEGORIZED(__completed_, "__exception_guard not completed with exceptions disabled"); + _LIBCPP_ASSERT_INTERNAL(__completed_, "__exception_guard not completed with exceptions disabled"); } private: diff --git a/libcxx/include/__utility/unreachable.h b/libcxx/include/__utility/unreachable.h index 49334decc8f6..d833f74c2e4f 100644 --- a/libcxx/include/__utility/unreachable.h +++ b/libcxx/include/__utility/unreachable.h @@ -19,7 +19,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_NORETURN _LIBCPP_HIDE_FROM_ABI inline void __libcpp_unreachable() { - _LIBCPP_ASSERT_UNCATEGORIZED(false, "std::unreachable() was reached"); + _LIBCPP_ASSERT_INTERNAL(false, "std::unreachable() was reached"); __builtin_unreachable(); } diff --git a/libcxx/include/print b/libcxx/include/print index 0f8e73f8eb5c..5e00fc87f47e 100644 --- a/libcxx/include/print +++ b/libcxx/include/print @@ -122,6 +122,8 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __encode(_OutIt&, char32_t) = delete; template requires __utf16_code_unit> _LIBCPP_HIDE_FROM_ABI constexpr void __encode(_OutIt& __out_it, char32_t __value) { + // [print.fun]/7 : "if `out` contains invalid code units, the behavior is undefined and implementations are encouraged + // to diagnose it". _LIBCPP_ASSERT_UNCATEGORIZED(__is_scalar_value(__value), "an invalid unicode scalar value results in invalid UTF-16"); if (__value < 0x10000) { @@ -137,6 +139,8 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __encode(_OutIt& __out_it, char32_t __value template requires __utf32_code_unit> _LIBCPP_HIDE_FROM_ABI constexpr void __encode(_OutIt& __out_it, char32_t __value) { + // [print.fun]/7 : "if `out` contains invalid code units, the behavior is undefined and implementations are encouraged + // to diagnose it". _LIBCPP_ASSERT_UNCATEGORIZED(__is_scalar_value(__value), "an invalid unicode scalar value results in invalid UTF-32"); *__out_it++ = __value; } @@ -214,7 +218,7 @@ _LIBCPP_HIDE_FROM_ABI inline bool __is_terminal(FILE* __stream) { template // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563). _LIBCPP_HIDE_FROM_ABI inline void __vprint_nonunicode(FILE* __stream, string_view __fmt, format_args __args, bool __write_nl) { - _LIBCPP_ASSERT_UNCATEGORIZED(__stream, "__stream is a valid pointer to an output C stream"); + _LIBCPP_ASSERT_NON_NULL(__stream, "__stream must be a valid pointer to an output C stream"); string __str = std::vformat(__fmt, __args); if (__write_nl) __str.push_back('\n'); @@ -290,7 +294,7 @@ __vprint_unicode([[maybe_unused]] FILE* __stream, [[maybe_unused]] string_view __fmt, [[maybe_unused]] format_args __args, [[maybe_unused]] bool __write_nl) { - _LIBCPP_ASSERT_UNCATEGORIZED(__stream, "__stream is a valid pointer to an output C stream"); + _LIBCPP_ASSERT_NON_NULL(__stream, "__stream must be a valid pointer to an output C stream"); // [print.fun] // 7 - Effects: If stream refers to a terminal capable of displaying diff --git a/libcxx/include/regex b/libcxx/include/regex index 061194cb2eba..b575a267583b 100644 --- a/libcxx/include/regex +++ b/libcxx/include/regex @@ -4587,28 +4587,36 @@ public: // element access: _LIBCPP_HIDE_FROM_ABI difference_type length(size_type __sub = 0) const { - _LIBCPP_ASSERT_UNCATEGORIZED(ready(), "match_results::length() called when not ready"); + // If the match results are not ready, this will return `0`. + _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::length() called when not ready"); return (*this)[__sub].length(); } _LIBCPP_HIDE_FROM_ABI difference_type position(size_type __sub = 0) const { - _LIBCPP_ASSERT_UNCATEGORIZED(ready(), "match_results::position() called when not ready"); + // If the match results are not ready, this will return the result of subtracting two default-constructed iterators + // (which is typically a well-defined operation). + _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::position() called when not ready"); return std::distance(__position_start_, (*this)[__sub].first); } _LIBCPP_HIDE_FROM_ABI string_type str(size_type __sub = 0) const { - _LIBCPP_ASSERT_UNCATEGORIZED(ready(), "match_results::str() called when not ready"); + // If the match results are not ready, this will return an empty string. + _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::str() called when not ready"); return (*this)[__sub].str(); } _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const { - _LIBCPP_ASSERT_UNCATEGORIZED(ready(), "match_results::operator[]() called when not ready"); + // If the match results are not ready, this call will be equivalent to calling this function with `__n >= size()`, + // returning an empty subrange. + _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::operator[]() called when not ready"); return __n < __matches_.size() ? __matches_[__n] : __unmatched_; } _LIBCPP_HIDE_FROM_ABI const_reference prefix() const { - _LIBCPP_ASSERT_UNCATEGORIZED(ready(), "match_results::prefix() called when not ready"); + // If the match results are not ready, this will return a default-constructed empty `__suffix_`. + _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::prefix() called when not ready"); return __prefix_; } _LIBCPP_HIDE_FROM_ABI const_reference suffix() const { - _LIBCPP_ASSERT_UNCATEGORIZED(ready(), "match_results::suffix() called when not ready"); + // If the match results are not ready, this will return a default-constructed empty `__suffix_`. + _LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::suffix() called when not ready"); return __suffix_; } @@ -4722,7 +4730,8 @@ _OutputIter match_results<_BidirectionalIterator, _Allocator>::format( const char_type* __fmt_first, const char_type* __fmt_last, regex_constants::match_flag_type __flags) const { - _LIBCPP_ASSERT_UNCATEGORIZED(ready(), "match_results::format() called when not ready"); + // Note: this duplicates a check in `vector::operator[]` but provides a better error message. + _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(ready(), "match_results::format() called when not ready"); if (__flags & regex_constants::format_sed) { for (; __fmt_first != __fmt_last; ++__fmt_first) { if (*__fmt_first == '&') diff --git a/libcxx/include/set b/libcxx/include/set index 08677a94054f..55ba8f8208be 100644 --- a/libcxx/include/set +++ b/libcxx/include/set @@ -769,13 +769,13 @@ public: #if _LIBCPP_STD_VER >= 17 _LIBCPP_HIDE_FROM_ABI insert_return_type insert(node_type&& __nh) { - _LIBCPP_ASSERT_UNCATEGORIZED(__nh.empty() || __nh.get_allocator() == get_allocator(), - "node_type with incompatible allocator passed to set::insert()"); + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(), + "node_type with incompatible allocator passed to set::insert()"); return __tree_.template __node_handle_insert_unique< node_type, insert_return_type>(std::move(__nh)); } _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) { - _LIBCPP_ASSERT_UNCATEGORIZED(__nh.empty() || __nh.get_allocator() == get_allocator(), - "node_type with incompatible allocator passed to set::insert()"); + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(), + "node_type with incompatible allocator passed to set::insert()"); return __tree_.template __node_handle_insert_unique(__hint, std::move(__nh)); } _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) { @@ -786,25 +786,25 @@ public: } template _LIBCPP_HIDE_FROM_ABI void merge(set& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_unique(__source.__tree_); } template _LIBCPP_HIDE_FROM_ABI void merge(set&& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_unique(__source.__tree_); } template _LIBCPP_HIDE_FROM_ABI void merge(multiset& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_unique(__source.__tree_); } template _LIBCPP_HIDE_FROM_ABI void merge(multiset&& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_unique(__source.__tree_); } @@ -1227,13 +1227,13 @@ public: #if _LIBCPP_STD_VER >= 17 _LIBCPP_HIDE_FROM_ABI iterator insert(node_type&& __nh) { - _LIBCPP_ASSERT_UNCATEGORIZED(__nh.empty() || __nh.get_allocator() == get_allocator(), - "node_type with incompatible allocator passed to multiset::insert()"); + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(), + "node_type with incompatible allocator passed to multiset::insert()"); return __tree_.template __node_handle_insert_multi(std::move(__nh)); } _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, node_type&& __nh) { - _LIBCPP_ASSERT_UNCATEGORIZED(__nh.empty() || __nh.get_allocator() == get_allocator(), - "node_type with incompatible allocator passed to multiset::insert()"); + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR(__nh.empty() || __nh.get_allocator() == get_allocator(), + "node_type with incompatible allocator passed to multiset::insert()"); return __tree_.template __node_handle_insert_multi(__hint, std::move(__nh)); } _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) { @@ -1244,25 +1244,25 @@ public: } template _LIBCPP_HIDE_FROM_ABI void merge(multiset& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_multi(__source.__tree_); } template _LIBCPP_HIDE_FROM_ABI void merge(multiset&& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_multi(__source.__tree_); } template _LIBCPP_HIDE_FROM_ABI void merge(set& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_multi(__source.__tree_); } template _LIBCPP_HIDE_FROM_ABI void merge(set&& __source) { - _LIBCPP_ASSERT_UNCATEGORIZED( + _LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR( __source.get_allocator() == get_allocator(), "merging container with incompatible allocator"); __tree_.__node_handle_merge_multi(__source.__tree_); } diff --git a/libcxx/src/filesystem/error.h b/libcxx/src/filesystem/error.h index b86f4ed41071..572cc73292a1 100644 --- a/libcxx/src/filesystem/error.h +++ b/libcxx/src/filesystem/error.h @@ -99,7 +99,7 @@ inline errc __win_err_to_errc(int err) { #endif // _LIBCPP_WIN32API inline error_code capture_errno() { - _LIBCPP_ASSERT_UNCATEGORIZED(errno != 0, "Expected errno to be non-zero"); + _LIBCPP_ASSERT_INTERNAL(errno != 0, "Expected errno to be non-zero"); return error_code(errno, generic_category()); } diff --git a/libcxx/src/filesystem/format_string.h b/libcxx/src/filesystem/format_string.h index 215d42421b2a..a44def86f53e 100644 --- a/libcxx/src/filesystem/format_string.h +++ b/libcxx/src/filesystem/format_string.h @@ -47,7 +47,7 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 0) string vformat_string(const ch size_t size_with_null = static_cast(ret) + 1; result.__resize_default_init(size_with_null - 1); ret = ::vsnprintf(&result[0], size_with_null, msg, ap); - _LIBCPP_ASSERT_UNCATEGORIZED(static_cast(ret) == (size_with_null - 1), "TODO"); + _LIBCPP_ASSERT_INTERNAL(static_cast(ret) == (size_with_null - 1), "TODO"); } return result; } diff --git a/libcxx/src/filesystem/posix_compat.h b/libcxx/src/filesystem/posix_compat.h index ec2de49960be..760cdb65dae1 100644 --- a/libcxx/src/filesystem/posix_compat.h +++ b/libcxx/src/filesystem/posix_compat.h @@ -318,8 +318,8 @@ inline int statvfs(const wchar_t* p, StatVFS* buf) { inline wchar_t* getcwd([[maybe_unused]] wchar_t* in_buf, [[maybe_unused]] size_t in_size) { // Only expected to be used with us allocating the buffer. - _LIBCPP_ASSERT_UNCATEGORIZED(in_buf == nullptr, "Windows getcwd() assumes in_buf==nullptr"); - _LIBCPP_ASSERT_UNCATEGORIZED(in_size == 0, "Windows getcwd() assumes in_size==0"); + _LIBCPP_ASSERT_INTERNAL(in_buf == nullptr, "Windows getcwd() assumes in_buf==nullptr"); + _LIBCPP_ASSERT_INTERNAL(in_size == 0, "Windows getcwd() assumes in_size==0"); size_t buff_size = MAX_PATH + 10; std::unique_ptr buff(static_cast(malloc(buff_size * sizeof(wchar_t))), &::free); @@ -338,7 +338,7 @@ inline wchar_t* getcwd([[maybe_unused]] wchar_t* in_buf, [[maybe_unused]] size_t inline wchar_t* realpath(const wchar_t* path, [[maybe_unused]] wchar_t* resolved_name) { // Only expected to be used with us allocating the buffer. - _LIBCPP_ASSERT_UNCATEGORIZED(resolved_name == nullptr, "Windows realpath() assumes a null resolved_name"); + _LIBCPP_ASSERT_INTERNAL(resolved_name == nullptr, "Windows realpath() assumes a null resolved_name"); WinHandle h(path, FILE_READ_ATTRIBUTES, 0); if (!h) { diff --git a/libcxx/src/include/to_chars_floating_point.h b/libcxx/src/include/to_chars_floating_point.h index 3110bc20e160..e4715d10d97d 100644 --- a/libcxx/src/include/to_chars_floating_point.h +++ b/libcxx/src/include/to_chars_floating_point.h @@ -269,7 +269,7 @@ to_chars_result _Floating_to_chars_hex_precision( // * Print the leading hexit, then mask it away. { const uint32_t _Nibble = static_cast(_Adjusted_mantissa >> _Adjusted_explicit_bits); - _LIBCPP_ASSERT_UNCATEGORIZED(_Nibble < 3, ""); + _LIBCPP_ASSERT_INTERNAL(_Nibble < 3, ""); const char _Leading_hexit = static_cast('0' + _Nibble); *_First++ = _Leading_hexit; @@ -288,12 +288,12 @@ to_chars_result _Floating_to_chars_hex_precision( int32_t _Number_of_bits_remaining = _Adjusted_explicit_bits; // 24 for float, 52 for double for (;;) { - _LIBCPP_ASSERT_UNCATEGORIZED(_Number_of_bits_remaining >= 4, ""); - _LIBCPP_ASSERT_UNCATEGORIZED(_Number_of_bits_remaining % 4 == 0, ""); + _LIBCPP_ASSERT_INTERNAL(_Number_of_bits_remaining >= 4, ""); + _LIBCPP_ASSERT_INTERNAL(_Number_of_bits_remaining % 4 == 0, ""); _Number_of_bits_remaining -= 4; const uint32_t _Nibble = static_cast(_Adjusted_mantissa >> _Number_of_bits_remaining); - _LIBCPP_ASSERT_UNCATEGORIZED(_Nibble < 16, ""); + _LIBCPP_ASSERT_INTERNAL(_Nibble < 16, ""); const char _Hexit = __itoa::_Charconv_digits[_Nibble]; *_First++ = _Hexit; @@ -415,12 +415,12 @@ to_chars_result _Floating_to_chars_hex_shortest( // '0' hexits, the same condition works (as we print the final hexit and mask it away); we don't need to test // _Number_of_bits_remaining. do { - _LIBCPP_ASSERT_UNCATEGORIZED(_Number_of_bits_remaining >= 4, ""); - _LIBCPP_ASSERT_UNCATEGORIZED(_Number_of_bits_remaining % 4 == 0, ""); + _LIBCPP_ASSERT_INTERNAL(_Number_of_bits_remaining >= 4, ""); + _LIBCPP_ASSERT_INTERNAL(_Number_of_bits_remaining % 4 == 0, ""); _Number_of_bits_remaining -= 4; const uint32_t _Nibble = static_cast(_Adjusted_mantissa >> _Number_of_bits_remaining); - _LIBCPP_ASSERT_UNCATEGORIZED(_Nibble < 16, ""); + _LIBCPP_ASSERT_INTERNAL(_Nibble < 16, ""); const char _Hexit = __itoa::_Charconv_digits[_Nibble]; if (_First == _Last) { @@ -940,13 +940,13 @@ to_chars_result _Floating_to_chars_general_precision( _Effective_precision = std::min(_Precision - (_Scientific_exponent_X + 1), _Max_fixed_precision); const to_chars_result _Buf_result = _Floating_to_chars_fixed_precision(_Buffer, std::end(_Buffer), _Value, _Effective_precision); - _LIBCPP_ASSERT_UNCATEGORIZED(_Buf_result.ec == errc{}, ""); + _LIBCPP_ASSERT_INTERNAL(_Buf_result.ec == errc{}, ""); _Significand_last = _Buf_result.ptr; } else { _Effective_precision = std::min(_Precision - 1, _Max_scientific_precision); const to_chars_result _Buf_result = _Floating_to_chars_scientific_precision(_Buffer, std::end(_Buffer), _Value, _Effective_precision); - _LIBCPP_ASSERT_UNCATEGORIZED(_Buf_result.ec == errc{}, ""); + _LIBCPP_ASSERT_INTERNAL(_Buf_result.ec == errc{}, ""); _Significand_last = std::find(_Buffer, _Buf_result.ptr, 'e'); _Exponent_first = _Significand_last; _Exponent_last = _Buf_result.ptr; @@ -992,7 +992,7 @@ to_chars_result _Floating_to_chars( char* _First, char* const _Last, _Floating _Value, const chars_format _Fmt, const int _Precision) noexcept { if constexpr (_Overload == _Floating_to_chars_overload::_Plain) { - _LIBCPP_ASSERT_UNCATEGORIZED(_Fmt == chars_format{}, ""); // plain overload must pass chars_format{} internally + _LIBCPP_ASSERT_INTERNAL(_Fmt == chars_format{}, ""); // plain overload must pass chars_format{} internally } else { _LIBCPP_ASSERT_UNCATEGORIZED(_Fmt == chars_format::general || _Fmt == chars_format::scientific || _Fmt == chars_format::fixed || _Fmt == chars_format::hex, diff --git a/libcxx/src/memory_resource.cpp b/libcxx/src/memory_resource.cpp index afd1b892086d..42c366893f73 100644 --- a/libcxx/src/memory_resource.cpp +++ b/libcxx/src/memory_resource.cpp @@ -230,7 +230,7 @@ public: } void* __allocate_in_new_chunk(memory_resource* upstream, size_t block_size, size_t chunk_size) { - _LIBCPP_ASSERT_UNCATEGORIZED(chunk_size % block_size == 0, ""); + _LIBCPP_ASSERT_INTERNAL(chunk_size % block_size == 0, ""); static_assert(__default_alignment >= alignof(std::max_align_t), ""); static_assert(__default_alignment >= alignof(__chunk_footer), ""); static_assert(__default_alignment >= alignof(__vacancy_header), ""); diff --git a/libcxx/src/strstream.cpp b/libcxx/src/strstream.cpp index a9b5989ec495..70374191c6ab 100644 --- a/libcxx/src/strstream.cpp +++ b/libcxx/src/strstream.cpp @@ -120,7 +120,7 @@ strstreambuf::int_type strstreambuf::overflow(int_type __c) { if (buf == nullptr) return int_type(EOF); if (old_size != 0) { - _LIBCPP_ASSERT_UNCATEGORIZED(eback(), "overflow copying from NULL"); + _LIBCPP_ASSERT_INTERNAL(eback(), "strstreambuf::overflow reallocating but the get area is a null pointer"); memcpy(buf, eback(), static_cast(old_size)); } ptrdiff_t ninp = gptr() - eback(); diff --git a/libcxx/src/system_error.cpp b/libcxx/src/system_error.cpp index 034b73c5480a..f518b480a278 100644 --- a/libcxx/src/system_error.cpp +++ b/libcxx/src/system_error.cpp @@ -68,7 +68,7 @@ __attribute__((unused)) const char* handle_strerror_r_return(int strerror_return if (new_errno == EINVAL) return ""; - _LIBCPP_ASSERT_UNCATEGORIZED(new_errno == ERANGE, "unexpected error from ::strerror_r"); + _LIBCPP_ASSERT_INTERNAL(new_errno == ERANGE, "unexpected error from ::strerror_r"); // FIXME maybe? 'strerror_buff_size' is likely to exceed the // maximum error size so ERANGE shouldn't be returned. std::abort(); diff --git a/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp b/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp index 8c14b84278cb..cf6b7bef3ebc 100644 --- a/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp +++ b/libcxx/test/libcxx/utilities/assert.exception_guard.no_exceptions.pass.cpp @@ -8,7 +8,7 @@ // REQUIRES: has-unix-headers // UNSUPPORTED: c++03 -// REQUIRES: libcpp-hardening-mode={{extensive|debug}} +// REQUIRES: libcpp-hardening-mode=debug // XFAIL: availability-verbose_abort-missing // ADDITIONAL_COMPILE_FLAGS: -fno-exceptions diff --git a/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp b/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp index 112b687cb17e..f95b83ff4eb3 100644 --- a/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp +++ b/libcxx/test/std/utilities/utility/utility.unreachable/assert.unreachable.pass.cpp @@ -8,7 +8,7 @@ // REQUIRES: has-unix-headers // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20 -// REQUIRES: libcpp-hardening-mode={{extensive|debug}} +// REQUIRES: libcpp-hardening-mode=debug // XFAIL: availability-verbose_abort-missing // Make sure that reaching std::unreachable() with assertions enabled triggers an assertion. -- GitLab From 4df566290751403f470fea3b27aa148ab1ddf144 Mon Sep 17 00:00:00 2001 From: hev Date: Sat, 6 Jan 2024 08:51:59 +0800 Subject: [PATCH 170/728] [clang] Add per-global code model attribute (#72078) This patch adds a per-global code model attribute, which can override the target's code model to access global variables. Currently, the code model attribute is only supported on LoongArch. This patch also maps GCC's code model names to LLVM's, which allows for better compatibility between the two compilers. Suggested-by: Arthur Eubanks Link: https://discourse.llvm.org/t/how-to-best-implement-code-model-overriding-for-certain-values/71816 Link: https://discourse.llvm.org/t/rfc-add-per-global-code-model-attribute/74944 --------- Signed-off-by: WANG Rui --- clang/include/clang/AST/Attr.h | 1 + clang/include/clang/Basic/Attr.td | 15 +++++ clang/include/clang/Basic/AttrDocs.td | 9 +++ .../clang/Basic/DiagnosticSemaKinds.td | 2 + clang/lib/CodeGen/CodeGenModule.cpp | 4 ++ clang/lib/Sema/SemaDeclAttr.cpp | 19 ++++++ clang/test/CodeGen/LoongArch/attributes.cpp | 34 ++++++++++ clang/test/Sema/attr-model.cpp | 64 +++++++++++++++++++ 8 files changed, 148 insertions(+) create mode 100644 clang/test/CodeGen/LoongArch/attributes.cpp create mode 100644 clang/test/Sema/attr-model.cpp diff --git a/clang/include/clang/AST/Attr.h b/clang/include/clang/AST/Attr.h index 1b831c9511e2..8e9b7ad8b468 100644 --- a/clang/include/clang/AST/Attr.h +++ b/clang/include/clang/AST/Attr.h @@ -25,6 +25,7 @@ #include "clang/Basic/Sanitizers.h" #include "clang/Basic/SourceLocation.h" #include "llvm/Frontend/HLSL/HLSLResource.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/VersionTuple.h" #include "llvm/Support/raw_ostream.h" diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index fda62aaae22c..d5eabaad4889 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -143,6 +143,11 @@ def ExternalGlobalVar : SubsetSubjectisLocalExternDecl()}], "external global variables">; +def NonTLSGlobalVar : SubsetSubjecthasGlobalStorage() && + S->getTLSKind() == 0}], + "non-TLS global variables">; + def InlineFunction : SubsetSubjectisInlineSpecified()}], "inline functions">; @@ -431,6 +436,7 @@ def TargetAArch64 : TargetArch<["aarch64", "aarch64_be", "aarch64_32"]>; def TargetAnyArm : TargetArch; def TargetAVR : TargetArch<["avr"]>; def TargetBPF : TargetArch<["bpfel", "bpfeb"]>; +def TargetLoongArch : TargetArch<["loongarch32", "loongarch64"]>; def TargetMips32 : TargetArch<["mips", "mipsel"]>; def TargetAnyMips : TargetArch<["mips", "mipsel", "mips64", "mips64el"]>; def TargetMSP430 : TargetArch<["msp430"]>; @@ -2738,6 +2744,15 @@ def PragmaClangTextSection : InheritableAttr { let Documentation = [InternalOnly]; } +def CodeModel : InheritableAttr, TargetSpecificAttr { + let Spellings = [GCC<"model">]; + let Args = [EnumArgument<"Model", "llvm::CodeModel::Model", + ["normal", "medium", "extreme"], ["Small", "Medium", "Large"], + /*opt=*/0, /*fake=*/0, /*isExternalType=*/1>]; + let Subjects = SubjectList<[NonTLSGlobalVar], ErrorDiag>; + let Documentation = [CodeModelDocs]; +} + def Sentinel : InheritableAttr { let Spellings = [GCC<"sentinel">]; let Args = [DefaultIntArgument<"Sentinel", 0>, diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index cd3dcf2ccf44..5416a0cbdd07 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -57,6 +57,15 @@ global variable or function should be in after translation. let Heading = "section, __declspec(allocate)"; } +def CodeModelDocs : Documentation { + let Category = DocCatVariable; + let Content = [{ +The ``model`` attribute allows overriding the translation unit's +code model (specified by ``-mcmodel``) for a specific global variable. + }]; + let Heading = "model"; +} + def UsedDocs : Documentation { let Category = DocCatFunction; let Content = [{ diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index e54f969c1903..d150e08d5f5e 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -3415,6 +3415,8 @@ def warn_objc_redundant_literal_use : Warning< def err_attr_tlsmodel_arg : Error<"tls_model must be \"global-dynamic\", " "\"local-dynamic\", \"initial-exec\" or \"local-exec\"">; +def err_attr_codemodel_arg : Error<"code model '%0' is not supported on this target">; + def err_aix_attr_unsupported_tls_model : Error<"TLS model '%0' is not yet supported on AIX">; def err_tls_var_aligned_over_maximum : Error< diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index d78f2594a237..4fd32337cccc 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -4869,6 +4869,10 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, isExternallyVisible(D->getLinkageAndVisibility().getLinkage())) GV->setSection(".cp.rodata"); + // Handle code model attribute + if (const auto *CMA = D->getAttr()) + GV->setCodeModel(CMA->getModel()); + // Check if we a have a const declaration with an initializer, we may be // able to emit it as available_externally to expose it's value to the // optimizer. diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 4a385a396fa6..d059b406ef86 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -3369,6 +3369,22 @@ static void handleSectionAttr(Sema &S, Decl *D, const ParsedAttr &AL) { } } +static void handleCodeModelAttr(Sema &S, Decl *D, const ParsedAttr &AL) { + StringRef Str; + SourceLocation LiteralLoc; + // Check that it is a string. + if (!S.checkStringLiteralArgumentAttr(AL, 0, Str, &LiteralLoc)) + return; + + llvm::CodeModel::Model CM; + if (!CodeModelAttr::ConvertStrToModel(Str, CM)) { + S.Diag(LiteralLoc, diag::err_attr_codemodel_arg) << Str; + return; + } + + D->addAttr(::new (S.Context) CodeModelAttr(S.Context, AL, CM)); +} + // This is used for `__declspec(code_seg("segname"))` on a decl. // `#pragma code_seg("segname")` uses checkSectionName() instead. static bool checkCodeSegName(Sema &S, SourceLocation LiteralLoc, @@ -9253,6 +9269,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, case ParsedAttr::AT_Section: handleSectionAttr(S, D, AL); break; + case ParsedAttr::AT_CodeModel: + handleCodeModelAttr(S, D, AL); + break; case ParsedAttr::AT_RandomizeLayout: handleRandomizeLayoutAttr(S, D, AL); break; diff --git a/clang/test/CodeGen/LoongArch/attributes.cpp b/clang/test/CodeGen/LoongArch/attributes.cpp new file mode 100644 index 000000000000..fb700ad30501 --- /dev/null +++ b/clang/test/CodeGen/LoongArch/attributes.cpp @@ -0,0 +1,34 @@ +// RUN: %clang_cc1 -emit-llvm -triple loongarch64 %s -o - | FileCheck %s + +// CHECK: @_ZL2v1 ={{.*}} global i32 0, code_model "small" +static int v1 __attribute__((model("normal"))); + +void use1() { + v1 = 1; +} + +// CHECK: @v2 ={{.*}} global i32 0, code_model "medium" +int v2 __attribute__((model("medium"))); + +// CHECK: @v3 ={{.*}} global float 0.000000e+00, code_model "large" +float v3 __attribute__((model("extreme"))); + +// CHECK: @_ZL2v4IiE ={{.*}} global i32 0, code_model "medium" +template +static T v4 __attribute__((model("medium"))); + +void use2() { + v4 = 1; +} + +struct S { + double d; +}; + +// CHECK: @v5 ={{.*}} global {{.*}}, code_model "medium" +S v5 __attribute__((model("medium"))); + +typedef void (*F)(); + +// CHECK: @v6 ={{.*}} global ptr null, code_model "large" +F v6 __attribute__((model("extreme"))); diff --git a/clang/test/Sema/attr-model.cpp b/clang/test/Sema/attr-model.cpp new file mode 100644 index 000000000000..898cc0393984 --- /dev/null +++ b/clang/test/Sema/attr-model.cpp @@ -0,0 +1,64 @@ +// RUN: %clang_cc1 -triple aarch64 -verify=expected,aarch64 -fsyntax-only %s +// RUN: %clang_cc1 -triple loongarch64 -verify=expected,loongarch64 -fsyntax-only %s +// RUN: %clang_cc1 -triple mips64 -verify=expected,mips64 -fsyntax-only %s +// RUN: %clang_cc1 -triple powerpc64 -verify=expected,powerpc64 -fsyntax-only %s +// RUN: %clang_cc1 -triple riscv64 -verify=expected,riscv64 -fsyntax-only %s +// RUN: %clang_cc1 -triple x86_64 -verify=expected,x86_64 -fsyntax-only %s + +#if defined(__loongarch__) && !__has_attribute(model) +#error "Should support model attribute" +#endif + +int a __attribute((model("tiny"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // loongarch64-error {{code model 'tiny' is not supported on this target}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} +int b __attribute((model("small"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // loongarch64-error {{code model 'small' is not supported on this target}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} +int c __attribute((model("normal"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} +int d __attribute((model("kernel"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // loongarch64-error {{code model 'kernel' is not supported on this target}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} +int e __attribute((model("medium"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} +int f __attribute((model("large"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // loongarch64-error {{code model 'large' is not supported on this target}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} +int g __attribute((model("extreme"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} + +void __attribute((model("extreme"))) h() {} // aarch64-warning {{unknown attribute 'model' ignored}} \ + // loongarch64-error {{'model' attribute only applies to non-TLS global variables}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} + +thread_local int i __attribute((model("extreme"))); // aarch64-warning {{unknown attribute 'model' ignored}} \ + // loongarch64-error {{'model' attribute only applies to non-TLS global variables}} \ + // mips64-warning {{unknown attribute 'model' ignored}} \ + // powerpc64-warning {{unknown attribute 'model' ignored}} \ + // riscv64-warning {{unknown attribute 'model' ignored}} \ + // x86_64-warning {{unknown attribute 'model' ignored}} -- GitLab From 2652243f19314cf0a7583402d37d28dbae9ec1e6 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 5 Jan 2024 16:53:48 -0800 Subject: [PATCH 171/728] [NFC][tsan] Move SkipInternalFrames into sanitizer_common (#77146) --- .../lib/sanitizer_common/sanitizer_common.h | 3 +++ .../sanitizer_symbolizer_report.cpp | 20 +++++++++++++++++++ compiler-rt/lib/tsan/rtl/tsan_report.cpp | 17 ++-------------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 6b327a4aa16f..7d9d61de8b61 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -32,6 +32,7 @@ struct AddressInfo; struct BufferedStackTrace; struct SignalContext; struct StackTrace; +struct SymbolizedStack; // Constants. const uptr kWordSize = SANITIZER_WORDSIZE / 8; @@ -393,6 +394,8 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info, // Same as above, but obtains AddressInfo by symbolizing top stack trace frame. void ReportErrorSummary(const char *error_type, const StackTrace *trace, const char *alt_tool_name = nullptr); +// Skips frames which we consider internal and not usefull to the users. +SymbolizedStack *SkipInternalFrames(SymbolizedStack *frames); void ReportMmapWriteExec(int prot, int mflags); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp index 3e4417ae3f57..ec60dd3e0d66 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -28,6 +28,26 @@ namespace __sanitizer { #if !SANITIZER_GO + +static bool FrameIsInternal(const SymbolizedStack *frame) { + if (!frame) + return true; + const char *file = frame->info.file; + const char *module = frame->info.module; + if (file && (internal_strstr(file, "/compiler-rt/lib/"))) + return true; + if (module && (internal_strstr(module, "libclang_rt."))) + return true; + return false; +} + +SymbolizedStack *SkipInternalFrames(SymbolizedStack *frames) { + for (SymbolizedStack *f = frames; f; f = f->next) + if (!FrameIsInternal(f)) + return f; + return nullptr; +} + void ReportErrorSummary(const char *error_type, const AddressInfo &info, const char *alt_tool_name) { if (!common_flags()->print_summary) return; diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp index c6b764bd8917..b1cee7ac8cbf 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp @@ -273,22 +273,9 @@ static ReportStack *ChooseSummaryStack(const ReportDesc *rep) { return 0; } -static bool FrameIsInternal(const SymbolizedStack *frame) { - if (!frame) - return true; - const char *file = frame->info.file; - const char *module = frame->info.module; - if (file && (internal_strstr(file, "/compiler-rt/lib/"))) - return true; - if (module && (internal_strstr(module, "libclang_rt."))) - return true; - return false; -} - static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) { - for (SymbolizedStack *f = frames; f; f = f->next) - if (!FrameIsInternal(f)) - return f; + if (SymbolizedStack *f = SkipInternalFrames(frames)) + return f; return frames; // Fallback to the top frame. } -- GitLab From 09e0d71ad34169e075e146e13a93b2d79f992e1d Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 5 Jan 2024 17:05:25 -0800 Subject: [PATCH 172/728] [ubsan] Drop terminal "in " from reports without functions (#77163) --- compiler-rt/lib/ubsan/ubsan_diag.cpp | 2 +- compiler-rt/test/ubsan/TestCases/Misc/print_summary.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler-rt/lib/ubsan/ubsan_diag.cpp b/compiler-rt/lib/ubsan/ubsan_diag.cpp index aac270415318..67e884e4916c 100644 --- a/compiler-rt/lib/ubsan/ubsan_diag.cpp +++ b/compiler-rt/lib/ubsan/ubsan_diag.cpp @@ -88,7 +88,7 @@ static void MaybeReportErrorSummary(Location Loc, ErrorType Type) { AI.file = internal_strdup(SLoc.getFilename()); AI.line = SLoc.getLine(); AI.column = SLoc.getColumn(); - AI.function = internal_strdup(""); // Avoid printing ?? as function name. + AI.function = nullptr; ReportErrorSummary(ErrorKind, AI, GetSanititizerToolName()); AI.Clear(); return; diff --git a/compiler-rt/test/ubsan/TestCases/Misc/print_summary.c b/compiler-rt/test/ubsan/TestCases/Misc/print_summary.c index b67a0614dd2f..17885094b6ad 100644 --- a/compiler-rt/test/ubsan/TestCases/Misc/print_summary.c +++ b/compiler-rt/test/ubsan/TestCases/Misc/print_summary.c @@ -2,8 +2,8 @@ // RUN: %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-DEFAULT // RUN: %env_ubsan_opts=print_summary=0 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-NO_SUMMARY -// CHECK-DEFAULT: SUMMARY: UndefinedBehaviorSanitizer: {{.*}} -// CHECK-NO_SUMMARY-NOT: SUMMARY: UndefinedBehaviorSanitizer: {{.*}} +// CHECK-DEFAULT: SUMMARY: UndefinedBehaviorSanitizer: {{.*}}print_summary.c{{[:0-9 ]*$}} +// CHECK-NO_SUMMARY-NOT: SUMMARY: UndefinedBehaviorSanitizer: int main(int argc, char **argv) { int arr[argc - 2]; -- GitLab From 40c07b559aa6ab4bac074c943967d3207bc07ae0 Mon Sep 17 00:00:00 2001 From: Haowei Wu Date: Fri, 5 Jan 2024 17:23:40 -0800 Subject: [PATCH 173/728] Revert "[libc++][streams] P1759R6: Native handles and file streams (#76632)" This reverts commit 255f95a40377677dd762df5a1aa65bcbb4f75c79, which contains a breaking libcxx test on Windows when using C++26 --- libcxx/docs/FeatureTestMacroTable.rst | 2 +- libcxx/docs/ReleaseNotes/18.rst | 1 - libcxx/docs/Status/Cxx2cPapers.csv | 2 +- libcxx/include/fstream | 50 ---------- libcxx/include/version | 2 +- libcxx/src/CMakeLists.txt | 1 - libcxx/src/fstream.cpp | 37 ------- .../native_handle.assert.pass.cpp | 32 ------ .../filebuf.members/native_handle.pass.cpp | 58 ----------- .../fstreams/filebuf/types.pass.cpp | 9 +- .../native_handle.assert.pass.cpp | 32 ------ .../fstream.members/native_handle.pass.cpp | 27 ------ .../fstreams/fstream/types.pass.cpp | 9 +- .../native_handle.assert.pass.cpp | 32 ------ .../ifstream.members/native_handle.pass.cpp | 27 ------ .../fstreams/ifstream/types.pass.cpp | 9 +- .../fstreams/native_handle_test_helpers.h | 97 ------------------- .../native_handle.assert.pass.cpp | 32 ------ .../ofstream.members/native_handle.pass.cpp | 27 ------ .../fstreams/ofstream/types.pass.cpp | 9 +- .../file.streams/fstreams/types.h | 10 -- .../fstream.version.compile.pass.cpp | 16 ++- .../version.version.compile.pass.cpp | 16 ++- .../generate_feature_test_macro_components.py | 1 + 24 files changed, 30 insertions(+), 508 deletions(-) delete mode 100644 libcxx/src/fstream.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp delete mode 100644 libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst index 893a3b13ca06..8ce5ec9f64ef 100644 --- a/libcxx/docs/FeatureTestMacroTable.rst +++ b/libcxx/docs/FeatureTestMacroTable.rst @@ -418,7 +418,7 @@ Status --------------------------------------------------- ----------------- ``__cpp_lib_freestanding_variant`` *unimplemented* --------------------------------------------------- ----------------- - ``__cpp_lib_fstream_native_handle`` ``202306L`` + ``__cpp_lib_fstream_native_handle`` *unimplemented* --------------------------------------------------- ----------------- ``__cpp_lib_function_ref`` *unimplemented* --------------------------------------------------- ----------------- diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst index 882f53b8d9f8..cae2347be5fd 100644 --- a/libcxx/docs/ReleaseNotes/18.rst +++ b/libcxx/docs/ReleaseNotes/18.rst @@ -59,7 +59,6 @@ Implemented Papers - P2909R4 - Fix formatting of code units as integers (Dude, where’s my ``char``?) - P2821R5 - span.at() - P0521R0 - Proposed Resolution for CA 14 (shared_ptr use_count/unique) -- P1759R6 - Native handles and file streams Improvements and New Features diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index 5701717f3976..fa4a112d1436 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -19,7 +19,7 @@ "`P2757R3 `__","LWG","Type-checking format args","Varna June 2023","","","|format|" "`P2637R3 `__","LWG","Member ``visit``","Varna June 2023","","","|format|" "`P2641R4 `__","CWG, LWG","Checking if a ``union`` alternative is active","Varna June 2023","","","" -"`P1759R6 `__","LWG","Native handles and file streams","Varna June 2023","|Complete|","18.0","" +"`P1759R6 `__","LWG","Native handles and file streams","Varna June 2023","","","" "`P2697R1 `__","LWG","Interfacing ``bitset`` with ``string_view``","Varna June 2023","|Complete|","18.0","" "`P1383R2 `__","LWG","More ``constexpr`` for ```` and ````","Varna June 2023","","","" "`P2734R0 `__","LWG","Adding the new SI prefixes","Varna June 2023","|Complete|","17.0","" diff --git a/libcxx/include/fstream b/libcxx/include/fstream index cd22b53efaad..7a4e15b55d56 100644 --- a/libcxx/include/fstream +++ b/libcxx/include/fstream @@ -73,7 +73,6 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; - using native_handle_type = typename basic_filebuf::native_handle_type; // Since C++26 basic_ifstream(); explicit basic_ifstream(const char* s, ios_base::openmode mode = ios_base::in); @@ -86,7 +85,6 @@ public: void swap(basic_ifstream& rhs); basic_filebuf* rdbuf() const; - native_handle_type native_handle() const noexcept; // Since C++26 bool is_open() const; void open(const char* s, ios_base::openmode mode = ios_base::in); void open(const string& s, ios_base::openmode mode = ios_base::in); @@ -112,7 +110,6 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; - using native_handle_type = typename basic_filebuf::native_handle_type; // Since C++26 basic_ofstream(); explicit basic_ofstream(const char* s, ios_base::openmode mode = ios_base::out); @@ -125,8 +122,6 @@ public: void swap(basic_ofstream& rhs); basic_filebuf* rdbuf() const; - native_handle_type native_handle() const noexcept; // Since C++26 - bool is_open() const; void open(const char* s, ios_base::openmode mode = ios_base::out); void open(const string& s, ios_base::openmode mode = ios_base::out); @@ -153,7 +148,6 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; - using native_handle_type = typename basic_filebuf::native_handle_type; // Since C++26 basic_fstream(); explicit basic_fstream(const char* s, ios_base::openmode mode = ios_base::in|ios_base::out); @@ -166,7 +160,6 @@ public: void swap(basic_fstream& rhs); basic_filebuf* rdbuf() const; - native_handle_type native_handle() const noexcept; // Since C++26 bool is_open() const; void open(const char* s, ios_base::openmode mode = ios_base::in|ios_base::out); void open(const string& s, ios_base::openmode mode = ios_base::in|ios_base::out); @@ -217,10 +210,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD -# if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_WIN32API) -_LIBCPP_EXPORTED_FROM_ABI void* __filebuf_windows_native_handle(FILE* __file); -# endif - template class _LIBCPP_TEMPLATE_VIS basic_filebuf : public basic_streambuf<_CharT, _Traits> { public: @@ -230,15 +219,6 @@ public: typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; typedef typename traits_type::state_type state_type; -# if _LIBCPP_STD_VER >= 26 -# if defined(_LIBCPP_WIN32API) - using native_handle_type = void*; // HANDLE -# elif __has_include() - using native_handle_type = int; // POSIX file descriptor -# else -# error "Provide a native file handle!" -# endif -# endif // 27.9.1.2 Constructors/destructor: basic_filebuf(); @@ -265,18 +245,6 @@ public: # endif _LIBCPP_HIDE_FROM_ABI basic_filebuf* __open(int __fd, ios_base::openmode __mode); basic_filebuf* close(); -# if _LIBCPP_STD_VER >= 26 - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { - _LIBCPP_ASSERT_UNCATEGORIZED(this->is_open(), "File must be opened"); -# if defined(_LIBCPP_WIN32API) - return std::__filebuf_windows_native_handle(__file_); -# elif __has_include() - return fileno(__file_); -# else -# error "Provide a way to determine the file native handle!" -# endif - } -# endif // _LIBCPP_STD_VER >= 26 _LIBCPP_HIDE_FROM_ABI inline static const char* __make_mdstring(ios_base::openmode __mode) _NOEXCEPT; @@ -1056,9 +1024,6 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; -# if _LIBCPP_STD_VER >= 26 - using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type; -# endif _LIBCPP_HIDE_FROM_ABI basic_ifstream(); _LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const char* __s, ios_base::openmode __mode = ios_base::in); @@ -1076,9 +1041,6 @@ public: _LIBCPP_HIDE_FROM_ABI void swap(basic_ifstream& __rhs); _LIBCPP_HIDE_FROM_ABI basic_filebuf* rdbuf() const; -# if _LIBCPP_STD_VER >= 26 - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); } -# endif _LIBCPP_HIDE_FROM_ABI bool is_open() const; void open(const char* __s, ios_base::openmode __mode = ios_base::in); # ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR @@ -1209,9 +1171,6 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; -# if _LIBCPP_STD_VER >= 26 - using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type; -# endif _LIBCPP_HIDE_FROM_ABI basic_ofstream(); _LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const char* __s, ios_base::openmode __mode = ios_base::out); @@ -1231,9 +1190,6 @@ public: _LIBCPP_HIDE_FROM_ABI void swap(basic_ofstream& __rhs); _LIBCPP_HIDE_FROM_ABI basic_filebuf* rdbuf() const; -# if _LIBCPP_STD_VER >= 26 - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); } -# endif _LIBCPP_HIDE_FROM_ABI bool is_open() const; void open(const char* __s, ios_base::openmode __mode = ios_base::out); # ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR @@ -1365,9 +1321,6 @@ public: typedef typename traits_type::int_type int_type; typedef typename traits_type::pos_type pos_type; typedef typename traits_type::off_type off_type; -# if _LIBCPP_STD_VER >= 26 - using native_handle_type = typename basic_filebuf<_CharT, _Traits>::native_handle_type; -# endif _LIBCPP_HIDE_FROM_ABI basic_fstream(); _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const char* __s, @@ -1392,9 +1345,6 @@ public: _LIBCPP_HIDE_FROM_ABI void swap(basic_fstream& __rhs); _LIBCPP_HIDE_FROM_ABI basic_filebuf* rdbuf() const; -# if _LIBCPP_STD_VER >= 26 - _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); } -# endif _LIBCPP_HIDE_FROM_ABI bool is_open() const; _LIBCPP_HIDE_FROM_ABI void open(const char* __s, ios_base::openmode __mode = ios_base::in | ios_base::out); # ifdef _LIBCPP_HAS_OPEN_WITH_WCHAR diff --git a/libcxx/include/version b/libcxx/include/version index c96647894dce..d3c2791a7d0b 100644 --- a/libcxx/include/version +++ b/libcxx/include/version @@ -496,7 +496,7 @@ __cpp_lib_within_lifetime 202306L // # define __cpp_lib_freestanding_optional 202311L // # define __cpp_lib_freestanding_string_view 202311L // # define __cpp_lib_freestanding_variant 202311L -# define __cpp_lib_fstream_native_handle 202306L +// # define __cpp_lib_fstream_native_handle 202306L // # define __cpp_lib_function_ref 202306L // # define __cpp_lib_hazard_pointer 202306L // # define __cpp_lib_linalg 202311L diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt index 96e7c6362d8c..329964a00136 100644 --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -84,7 +84,6 @@ endif() if (LIBCXX_ENABLE_LOCALIZATION) list(APPEND LIBCXX_SOURCES - fstream.cpp include/sso_allocator.h ios.cpp ios.instantiations.cpp diff --git a/libcxx/src/fstream.cpp b/libcxx/src/fstream.cpp deleted file mode 100644 index 55a4442b9c78..000000000000 --- a/libcxx/src/fstream.cpp +++ /dev/null @@ -1,37 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include <__config> -#include -#include - -#if defined(_LIBCPP_WIN32API) -# define WIN32_LEAN_AND_MEAN -# define NOMINMAX -# include -# include -#endif - -_LIBCPP_BEGIN_NAMESPACE_STD - -#if defined(_LIBCPP_WIN32API) - -// Confirm that `HANDLE` is `void*` as implemented in `basic_filebuf` -static_assert(std::same_as); - -_LIBCPP_EXPORTED_FROM_ABI void* __filebuf_windows_native_handle(FILE* __file) noexcept { - // https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/get-osfhandle?view=msvc-170 - intptr_t __handle = _get_osfhandle(fileno(__file)); - if (__handle == -1) - return nullptr; - return reinterpret_cast(__handle); -} - -#endif - -_LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp deleted file mode 100644 index 858f32e72165..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.assert.pass.cpp +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// REQUIRES: has-unix-headers -// REQUIRES: libcpp-hardening-mode={{extensive|debug}} -// XFAIL: availability-verbose_abort-missing - -// - -// class basic_filebuf; - -// native_handle_type native_handle() const noexcept; - -#include - -#include "../native_handle_test_helpers.h" - -int main(int, char**) { - test_native_handle_assertion>(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_assertion>(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp deleted file mode 100644 index 22cc63e909c5..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/native_handle.pass.cpp +++ /dev/null @@ -1,58 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// - -// class basic_filebuf; - -// native_handle_type native_handle() const noexcept; - -#include -#include -#include -#include - -#include "platform_support.h" -#include "test_macros.h" -#include "../native_handle_test_helpers.h" - -template -void test() { - std::basic_filebuf f; - std::filesystem::path p = get_temp_file_name(); - - // non-const - { - assert(f.open(p, std::ios_base::in) != nullptr); - std::same_as decltype(auto) handle = f.native_handle(); - assert(is_handle_valid(handle)); - f.close(); - assert(!is_handle_valid(handle)); - static_assert(noexcept(f.native_handle())); - } - // const - { - assert(f.open(p, std::ios_base::in) != nullptr); - std::same_as decltype(auto) const_handle = std::as_const(f).native_handle(); - assert(is_handle_valid(const_handle)); - f.close(); - assert(!is_handle_valid(const_handle)); - static_assert(noexcept(std::as_const(f).native_handle())); - } -} - -int main(int, char**) { - test(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp index 53a5f78d5eff..ad4249f4d809 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf/types.pass.cpp @@ -23,7 +23,6 @@ #include #include "test_macros.h" -#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -33,12 +32,6 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); -#if TEST_STD_VER >= 26 - test_native_handle_type< std::basic_filebuf>(); -# ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_type< std::basic_filebuf>(); -# endif -#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp deleted file mode 100644 index 2e6cadbc64e3..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.assert.pass.cpp +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// REQUIRES: has-unix-headers -// REQUIRES: libcpp-hardening-mode={{extensive|debug}} -// XFAIL: availability-verbose_abort-missing - -// - -// class basic_fstream; - -// native_handle_type native_handle() const noexcept; - -#include - -#include "../native_handle_test_helpers.h" - -int main(int, char**) { - test_native_handle_assertion>(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_assertion>(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp deleted file mode 100644 index a7229512385a..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/fstream.members/native_handle.pass.cpp +++ /dev/null @@ -1,27 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// - -// class basic_fstream; - -// native_handle_type native_handle() const noexcept; - -#include "test_macros.h" -#include "../native_handle_test_helpers.h" - -int main(int, char**) { - test_native_handle>(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle>(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp index 0e275c53ed18..9274b1854bf8 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/fstream/types.pass.cpp @@ -23,7 +23,6 @@ #include #include "test_macros.h" -#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -33,12 +32,6 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); -#if TEST_STD_VER >= 26 - test_native_handle_type< std::basic_fstream>(); -# ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_type< std::basic_fstream>(); -# endif -#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp deleted file mode 100644 index 5e55b7303409..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.assert.pass.cpp +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// REQUIRES: has-unix-headers -// REQUIRES: libcpp-hardening-mode={{extensive|debug}} -// XFAIL: availability-verbose_abort-missing - -// - -// class basic_ifstream; - -// native_handle_type native_handle() const noexcept; - -#include - -#include "../native_handle_test_helpers.h" - -int main(int, char**) { - test_native_handle_assertion>(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_assertion>(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp deleted file mode 100644 index c2aff949c8fa..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/native_handle.pass.cpp +++ /dev/null @@ -1,27 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// - -// class basic_ifstream; - -// native_handle_type native_handle() const noexcept; - -#include "test_macros.h" -#include "../native_handle_test_helpers.h" - -int main(int, char**) { - test_native_handle>(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle>(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp index 580e1ed22cba..15a044021f64 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream/types.pass.cpp @@ -23,7 +23,6 @@ #include #include "test_macros.h" -#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -33,12 +32,6 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); -#if TEST_STD_VER >= 26 - test_native_handle_type< std::basic_ifstream>(); -# ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_type< std::basic_ifstream>(); -# endif -#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h b/libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h deleted file mode 100644 index 40bb2fe19487..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/native_handle_test_helpers.h +++ /dev/null @@ -1,97 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TEST_HELPERS_H -#define TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TEST_HELPERS_H - -#include -#include -#include -#include -#include -#include -#include - -#if defined(_WIN32) -# include -# include -#else -# include -#endif - -#include "platform_support.h" -#include "test_macros.h" -#include "types.h" - -#if TEST_STD_VER >= 26 - -# include "check_assertion.h" - -inline bool is_handle_valid(NativeHandleT handle) { -# if defined(_WIN32) - BY_HANDLE_FILE_INFORMATION fileInformation; - return GetFileInformationByHandle(handle, &fileInformation)); -# elif __has_include() // POSIX - return fcntl(handle, F_GETFL) != -1 || errno != EBADF; -# else -# error "Provide a native file handle!" -# endif -} - -template -inline void test_native_handle() { - static_assert( - std::is_same_v::native_handle_type, typename StreamT::native_handle_type>); - - StreamT f; - std::filesystem::path p = get_temp_file_name(); - - // non-const - { - f.open(p); - std::same_as decltype(auto) handle = f.native_handle(); - assert(is_handle_valid(handle)); - assert(f.rdbuf()->native_handle() == handle); - assert(std::as_const(f).rdbuf()->native_handle() == handle); - f.close(); - assert(!is_handle_valid(handle)); - static_assert(noexcept(f.native_handle())); - } - // const - { - f.open(p); - std::same_as decltype(auto) const_handle = std::as_const(f).native_handle(); - assert(is_handle_valid(const_handle)); - assert(f.rdbuf()->native_handle() == const_handle); - assert(std::as_const(f).rdbuf()->native_handle() == const_handle); - f.close(); - assert(!is_handle_valid(const_handle)); - static_assert(noexcept(std::as_const(f).native_handle())); - } -} - -template -inline void test_native_handle_assertion() { - StreamT f; - - // non-const - TEST_LIBCPP_ASSERT_FAILURE(f.native_handle(), "File must be opened"); - // const - TEST_LIBCPP_ASSERT_FAILURE(std::as_const(f).native_handle(), "File must be opened"); -} - -template -inline void test_native_handle_type() { - static_assert(std::is_trivially_copyable_v); - static_assert(std::semiregular); - static_assert(std::is_same_v); -} - -#endif // #if TEST_STD_VER >= 26 - -#endif // TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TEST_HELPERS_H diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp deleted file mode 100644 index d42aec1e99b9..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.assert.pass.cpp +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// REQUIRES: has-unix-headers -// REQUIRES: libcpp-hardening-mode={{extensive|debug}} -// XFAIL: availability-verbose_abort-missing - -// - -// class basic_ofstream; - -// native_handle_type native_handle() const noexcept; - -#include - -#include "../native_handle_test_helpers.h" - -int main(int, char**) { - test_native_handle_assertion>(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_assertion>(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp deleted file mode 100644 index 88b3a00934cc..000000000000 --- a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/native_handle.pass.cpp +++ /dev/null @@ -1,27 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23 - -// - -// class basic_ofstream; - -// native_handle_type native_handle() const noexcept; - -#include "test_macros.h" -#include "../native_handle_test_helpers.h" - -int main(int, char**) { - test_native_handle>(); -#ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle>(); -#endif - - return 0; -} diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp index 242b4610af4a..0986d5907bac 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ofstream/types.pass.cpp @@ -23,7 +23,6 @@ #include #include "test_macros.h" -#include "../native_handle_test_helpers.h" int main(int, char**) { @@ -33,12 +32,6 @@ int main(int, char**) static_assert((std::is_same::int_type, std::char_traits::int_type>::value), ""); static_assert((std::is_same::pos_type, std::char_traits::pos_type>::value), ""); static_assert((std::is_same::off_type, std::char_traits::off_type>::value), ""); -#if TEST_STD_VER >= 26 - test_native_handle_type< std::basic_ofstream>(); -# ifndef TEST_HAS_NO_WIDE_CHARACTERS - test_native_handle_type< std::basic_ofstream>(); -# endif -#endif - return 0; + return 0; } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/types.h b/libcxx/test/std/input.output/file.streams/fstreams/types.h index 29c8ad2d06c4..b919ba201853 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/types.h +++ b/libcxx/test/std/input.output/file.streams/fstreams/types.h @@ -80,14 +80,4 @@ struct LibraryDefaultBuffer { void operator()(std::basic_ifstream&) const {} }; -#if TEST_STD_VER >= 26 -# if defined(_WIN32) -using NativeHandleT = void*; // HANDLE -# elif __has_include() -using NativeHandleT = int; // POSIX file descriptor -# else -# error "Provide a native file handle!" -# endif -#endif - #endif // TEST_STD_INPUT_OUTPUT_FILE_STREAMS_FSTREAMS_TYPES_H diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp index eab0313b2c1e..981f5420ff7e 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp @@ -56,11 +56,17 @@ #elif TEST_STD_VER > 23 -# ifndef __cpp_lib_fstream_native_handle -# error "__cpp_lib_fstream_native_handle should be defined in c++26" -# endif -# if __cpp_lib_fstream_native_handle != 202306L -# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" +# if !defined(_LIBCPP_VERSION) +# ifndef __cpp_lib_fstream_native_handle +# error "__cpp_lib_fstream_native_handle should be defined in c++26" +# endif +# if __cpp_lib_fstream_native_handle != 202306L +# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" +# endif +# else // _LIBCPP_VERSION +# ifdef __cpp_lib_fstream_native_handle +# error "__cpp_lib_fstream_native_handle should not be defined because it is unimplemented in libc++!" +# endif # endif #endif // TEST_STD_VER > 23 diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp index d5a0839b30f8..3b7f2d26feeb 100644 --- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp @@ -6511,11 +6511,17 @@ # endif # endif -# ifndef __cpp_lib_fstream_native_handle -# error "__cpp_lib_fstream_native_handle should be defined in c++26" -# endif -# if __cpp_lib_fstream_native_handle != 202306L -# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" +# if !defined(_LIBCPP_VERSION) +# ifndef __cpp_lib_fstream_native_handle +# error "__cpp_lib_fstream_native_handle should be defined in c++26" +# endif +# if __cpp_lib_fstream_native_handle != 202306L +# error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26" +# endif +# else // _LIBCPP_VERSION +# ifdef __cpp_lib_fstream_native_handle +# error "__cpp_lib_fstream_native_handle should not be defined because it is unimplemented in libc++!" +# endif # endif # if !defined(_LIBCPP_VERSION) diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py index 8ee92909dfa5..3ad5170d73ff 100755 --- a/libcxx/utils/generate_feature_test_macro_components.py +++ b/libcxx/utils/generate_feature_test_macro_components.py @@ -570,6 +570,7 @@ feature_test_macros = [ "name": "__cpp_lib_fstream_native_handle", "values": {"c++26": 202306}, # P1759R6 Native handles and file streams "headers": ["fstream"], + "unimplemented": True, }, { "name": "__cpp_lib_function_ref", -- GitLab From 1c43e64d7072bba3e6199999184f8323045684ac Mon Sep 17 00:00:00 2001 From: paperchalice Date: Sat, 6 Jan 2024 09:25:15 +0800 Subject: [PATCH 174/728] [CodeGen] Port `ShadowStackGCLowering` to new pass manager (#75324) IIUC the new pass system was designed with parallelism. This pass needs to add some global variables into the current module, this is not allowed by [WritingAnLLVMPass](https://llvm.org/docs/WritingAnLLVMPass.html#the-functionpass-class), so convert it to module pass, see FIXME in `GetFrameMap`. Therefore, this will trigger assertion in `CodeGenPassBuilder`: https://github.com/llvm/llvm-project/blob/effd47ed45e3badd756103346a7c3b9e1e939e5e/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h#L200-L207 Will fix it in future. --- .../include/llvm/CodeGen/CodeGenPassBuilder.h | 4 + .../llvm/CodeGen/MachinePassRegistry.def | 2 +- .../llvm/CodeGen/ShadowStackGCLowering.h | 24 +++++ llvm/lib/CodeGen/ShadowStackGCLowering.cpp | 97 +++++++++++++------ llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassRegistry.def | 1 + 6 files changed, 97 insertions(+), 32 deletions(-) create mode 100644 llvm/include/llvm/CodeGen/ShadowStackGCLowering.h diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index a7cbb0910baa..2100c30aad11 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -38,6 +38,7 @@ #include "llvm/CodeGen/ReplaceWithVeclib.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/SelectOptimize.h" +#include "llvm/CodeGen/ShadowStackGCLowering.h" #include "llvm/CodeGen/SjLjEHPrepare.h" #include "llvm/CodeGen/UnreachableBlockElim.h" #include "llvm/CodeGen/WasmEHPrepare.h" @@ -642,6 +643,9 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { // Run GC lowering passes for builtin collectors // TODO: add a pass insertion point here addPass(GCLoweringPass()); + // FIXME: `ShadowStackGCLoweringPass` now is a + // module pass, so it will trigger assertion. + // See comment of `AddingFunctionPasses` addPass(ShadowStackGCLoweringPass()); addPass(LowerConstantIntrinsicsPass()); diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index f950dfae7e33..b1b8ee8df29d 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -26,6 +26,7 @@ MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC)) MODULE_PASS("pre-isel-intrinsic-lowering", PreISelIntrinsicLoweringPass, ()) MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass, ()) MODULE_PASS("lower-emutls", LowerEmuTLSPass, ()) +MODULE_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) #undef MODULE_PASS #ifndef FUNCTION_ANALYSIS @@ -133,7 +134,6 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ()) DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ()) DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ()) -DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ()) #undef DUMMY_FUNCTION_PASS diff --git a/llvm/include/llvm/CodeGen/ShadowStackGCLowering.h b/llvm/include/llvm/CodeGen/ShadowStackGCLowering.h new file mode 100644 index 000000000000..1586c6cf545b --- /dev/null +++ b/llvm/include/llvm/CodeGen/ShadowStackGCLowering.h @@ -0,0 +1,24 @@ +//===- llvm/CodeGen/ShadowStackGCLowering.h ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_SHADOWSTACKGCLOWERING_H +#define LLVM_CODEGEN_SHADOWSTACKGCLOWERING_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class ShadowStackGCLoweringPass + : public PassInfoMixin { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_SHADOWSTACKGCLOWERING_H diff --git a/llvm/lib/CodeGen/ShadowStackGCLowering.cpp b/llvm/lib/CodeGen/ShadowStackGCLowering.cpp index d4840d117110..232e5e2bb886 100644 --- a/llvm/lib/CodeGen/ShadowStackGCLowering.cpp +++ b/llvm/lib/CodeGen/ShadowStackGCLowering.cpp @@ -15,9 +15,11 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/ShadowStackGCLowering.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/DomTreeUpdater.h" +#include "llvm/CodeGen/GCMetadata.h" #include "llvm/CodeGen/Passes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" @@ -50,7 +52,7 @@ using namespace llvm; namespace { -class ShadowStackGCLowering : public FunctionPass { +class ShadowStackGCLoweringImpl { /// RootChain - This is the global linked-list that contains the chain of GC /// roots. GlobalVariable *Head = nullptr; @@ -64,13 +66,10 @@ class ShadowStackGCLowering : public FunctionPass { std::vector> Roots; public: - static char ID; - - ShadowStackGCLowering(); + ShadowStackGCLoweringImpl() = default; - bool doInitialization(Module &M) override; - void getAnalysisUsage(AnalysisUsage &AU) const override; - bool runOnFunction(Function &F) override; + bool doInitialization(Module &M); + bool runOnFunction(Function &F, DomTreeUpdater *DTU); private: bool IsNullValue(Value *V); @@ -86,8 +85,51 @@ private: const char *Name); }; +class ShadowStackGCLowering : public FunctionPass { + ShadowStackGCLoweringImpl Impl; + +public: + static char ID; + + ShadowStackGCLowering(); + + bool doInitialization(Module &M) override { return Impl.doInitialization(M); } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addPreserved(); + } + bool runOnFunction(Function &F) override { + std::optional DTU; + if (auto *DTWP = getAnalysisIfAvailable()) + DTU.emplace(DTWP->getDomTree(), DomTreeUpdater::UpdateStrategy::Lazy); + return Impl.runOnFunction(F, DTU ? &*DTU : nullptr); + } +}; + } // end anonymous namespace +PreservedAnalyses ShadowStackGCLoweringPass::run(Module &M, + ModuleAnalysisManager &MAM) { + auto &Map = MAM.getResult(M); + if (Map.StrategyMap.contains("shadow-stack")) + return PreservedAnalyses::all(); + + ShadowStackGCLoweringImpl Impl; + bool Changed = Impl.doInitialization(M); + for (auto &F : M) { + auto &FAM = + MAM.getResult(M).getManager(); + auto *DT = FAM.getCachedResult(F); + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); + Changed |= Impl.runOnFunction(F, DT ? &DTU : nullptr); + } + + if (!Changed) + return PreservedAnalyses::all(); + PreservedAnalyses PA; + PA.preserve(); + return PA; +} + char ShadowStackGCLowering::ID = 0; char &llvm::ShadowStackGCLoweringID = ShadowStackGCLowering::ID; @@ -104,7 +146,7 @@ ShadowStackGCLowering::ShadowStackGCLowering() : FunctionPass(ID) { initializeShadowStackGCLoweringPass(*PassRegistry::getPassRegistry()); } -Constant *ShadowStackGCLowering::GetFrameMap(Function &F) { +Constant *ShadowStackGCLoweringImpl::GetFrameMap(Function &F) { // doInitialization creates the abstract type of this value. Type *VoidPtr = PointerType::getUnqual(F.getContext()); @@ -158,7 +200,7 @@ Constant *ShadowStackGCLowering::GetFrameMap(Function &F) { return ConstantExpr::getGetElementPtr(FrameMap->getType(), GV, GEPIndices); } -Type *ShadowStackGCLowering::GetConcreteStackEntryType(Function &F) { +Type *ShadowStackGCLoweringImpl::GetConcreteStackEntryType(Function &F) { // doInitialization creates the generic version of this type. std::vector EltTys; EltTys.push_back(StackEntryTy); @@ -170,7 +212,7 @@ Type *ShadowStackGCLowering::GetConcreteStackEntryType(Function &F) { /// doInitialization - If this module uses the GC intrinsics, find them now. If /// not, exit fast. -bool ShadowStackGCLowering::doInitialization(Module &M) { +bool ShadowStackGCLoweringImpl::doInitialization(Module &M) { bool Active = false; for (Function &F : M) { if (F.hasGC() && F.getGC() == "shadow-stack") { @@ -224,13 +266,13 @@ bool ShadowStackGCLowering::doInitialization(Module &M) { return true; } -bool ShadowStackGCLowering::IsNullValue(Value *V) { +bool ShadowStackGCLoweringImpl::IsNullValue(Value *V) { if (Constant *C = dyn_cast(V)) return C->isNullValue(); return false; } -void ShadowStackGCLowering::CollectRoots(Function &F) { +void ShadowStackGCLoweringImpl::CollectRoots(Function &F) { // FIXME: Account for original alignment. Could fragment the root array. // Approach 1: Null initialize empty slots at runtime. Yuck. // Approach 2: Emit a map of the array instead of just a count. @@ -258,11 +300,10 @@ void ShadowStackGCLowering::CollectRoots(Function &F) { Roots.insert(Roots.begin(), MetaRoots.begin(), MetaRoots.end()); } -GetElementPtrInst *ShadowStackGCLowering::CreateGEP(LLVMContext &Context, - IRBuilder<> &B, Type *Ty, - Value *BasePtr, int Idx, - int Idx2, - const char *Name) { +GetElementPtrInst * +ShadowStackGCLoweringImpl::CreateGEP(LLVMContext &Context, IRBuilder<> &B, + Type *Ty, Value *BasePtr, int Idx, + int Idx2, const char *Name) { Value *Indices[] = {ConstantInt::get(Type::getInt32Ty(Context), 0), ConstantInt::get(Type::getInt32Ty(Context), Idx), ConstantInt::get(Type::getInt32Ty(Context), Idx2)}; @@ -273,9 +314,11 @@ GetElementPtrInst *ShadowStackGCLowering::CreateGEP(LLVMContext &Context, return dyn_cast(Val); } -GetElementPtrInst *ShadowStackGCLowering::CreateGEP(LLVMContext &Context, - IRBuilder<> &B, Type *Ty, Value *BasePtr, - int Idx, const char *Name) { +GetElementPtrInst *ShadowStackGCLoweringImpl::CreateGEP(LLVMContext &Context, + IRBuilder<> &B, + Type *Ty, + Value *BasePtr, int Idx, + const char *Name) { Value *Indices[] = {ConstantInt::get(Type::getInt32Ty(Context), 0), ConstantInt::get(Type::getInt32Ty(Context), Idx)}; Value *Val = B.CreateGEP(Ty, BasePtr, Indices, Name); @@ -285,12 +328,9 @@ GetElementPtrInst *ShadowStackGCLowering::CreateGEP(LLVMContext &Context, return dyn_cast(Val); } -void ShadowStackGCLowering::getAnalysisUsage(AnalysisUsage &AU) const { - AU.addPreserved(); -} - /// runOnFunction - Insert code to maintain the shadow stack. -bool ShadowStackGCLowering::runOnFunction(Function &F) { +bool ShadowStackGCLoweringImpl::runOnFunction(Function &F, + DomTreeUpdater *DTU) { // Quick exit for functions that do not use the shadow stack GC. if (!F.hasGC() || F.getGC() != "shadow-stack") return false; @@ -305,10 +345,6 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) { if (Roots.empty()) return false; - std::optional DTU; - if (auto *DTWP = getAnalysisIfAvailable()) - DTU.emplace(DTWP->getDomTree(), DomTreeUpdater::UpdateStrategy::Lazy); - // Build the constant map and figure the type of the shadow stack entry. Value *FrameMap = GetFrameMap(F); Type *ConcreteStackEntryTy = GetConcreteStackEntryType(F); @@ -359,8 +395,7 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) { AtEntry.CreateStore(NewHeadVal, Head); // For each instruction that escapes... - EscapeEnumerator EE(F, "gc_cleanup", /*HandleExceptions=*/true, - DTU ? &*DTU : nullptr); + EscapeEnumerator EE(F, "gc_cleanup", /*HandleExceptions=*/true, DTU); while (IRBuilder<> *AtExit = EE.Next()) { // Pop the entry from the shadow stack. Don't reuse CurrentHead from // AtEntry, since that would make the value live for the entire function. diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 439f749bda8b..649451edc0e2 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -86,6 +86,7 @@ #include "llvm/CodeGen/LowerEmuTLS.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/SelectOptimize.h" +#include "llvm/CodeGen/ShadowStackGCLowering.h" #include "llvm/CodeGen/SjLjEHPrepare.h" #include "llvm/CodeGen/TypePromotion.h" #include "llvm/CodeGen/WasmEHPrepare.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 82ce040c6496..eaa7c3fc8924 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -128,6 +128,7 @@ MODULE_PASS("sanmd-module", SanitizerBinaryMetadataPass()) MODULE_PASS("scc-oz-module-inliner", buildInlinerPipeline(OptimizationLevel::Oz, ThinOrFullLTOPhase::None)) +MODULE_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass()) MODULE_PASS("strip", StripSymbolsPass()) MODULE_PASS("strip-dead-debug-info", StripDeadDebugInfoPass()) MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass()) -- GitLab From ba1f4c6c50205bb7c6937c236084987f8669191c Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Sat, 6 Jan 2024 01:31:06 +0000 Subject: [PATCH 175/728] [gn build] Port 40c07b559aa6 --- llvm/utils/gn/secondary/libcxx/src/BUILD.gn | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn index 5ba8a7cf4b91..c1211bb384d3 100644 --- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn @@ -122,7 +122,6 @@ cxx_sources = [ "condition_variable_destructor.cpp", "error_category.cpp", "exception.cpp", - "fstream.cpp", "functional.cpp", "future.cpp", "hash.cpp", @@ -146,7 +145,6 @@ cxx_sources = [ "iostream.cpp", "legacy_pointer_safety.cpp", "locale.cpp", - "ostream.cpp", "memory.cpp", "memory_resource.cpp", "mutex.cpp", @@ -155,6 +153,7 @@ cxx_sources = [ "new_handler.cpp", "new_helpers.cpp", "optional.cpp", + "ostream.cpp", "print.cpp", "random.cpp", "random_shuffle.cpp", -- GitLab From fc6b5666db92e0c1701177aee885db66296e5a86 Mon Sep 17 00:00:00 2001 From: Bill Wendling <5993918+bwendling@users.noreply.github.com> Date: Fri, 5 Jan 2024 18:08:53 -0800 Subject: [PATCH 176/728] [NFC][ObjectSizeOffset] Use classes instead of std::pair (#76882) The use of std::pair makes the values it holds opaque. Using classes improves this while keeping the POD aspect of a std::pair. As a nice addition, the "known" functions held inappropriately in the Visitor classes can now properly reside in the value classes. :-) --- llvm/include/llvm/Analysis/MemoryBuiltins.h | 171 ++++++---- llvm/lib/Analysis/MemoryBuiltins.cpp | 308 +++++++++--------- .../Transforms/IPO/AttributorAttributes.cpp | 6 +- .../Instrumentation/AddressSanitizer.cpp | 12 +- .../Instrumentation/BoundsChecking.cpp | 8 +- 5 files changed, 272 insertions(+), 233 deletions(-) diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h index 827b5081b2ce..d080a5956a4d 100644 --- a/llvm/include/llvm/Analysis/MemoryBuiltins.h +++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h @@ -187,80 +187,125 @@ Value *lowerObjectSizeCall( const TargetLibraryInfo *TLI, AAResults *AA, bool MustSucceed, SmallVectorImpl *InsertedInstructions = nullptr); -using SizeOffsetType = std::pair; +/// SizeOffsetType - A base template class for the object size visitors. Used +/// here as a self-documenting way to handle the values rather than using a +/// \p std::pair. +template class SizeOffsetType { +public: + T Size; + T Offset; + + SizeOffsetType() = default; + SizeOffsetType(T Size, T Offset) : Size(Size), Offset(Offset) {} + + bool knownSize() const { return C::known(Size); } + bool knownOffset() const { return C::known(Offset); } + bool anyKnown() const { return knownSize() || knownOffset(); } + bool bothKnown() const { return knownSize() && knownOffset(); } + + bool operator==(const SizeOffsetType &RHS) const { + return Size == RHS.Size && Offset == RHS.Offset; + } + bool operator!=(const SizeOffsetType &RHS) const { + return !(*this == RHS); + } +}; + +/// SizeOffsetAPInt - Used by \p ObjectSizeOffsetVisitor, which works with +/// \p APInts. +class SizeOffsetAPInt : public SizeOffsetType { + friend class SizeOffsetType; + static bool known(APInt V) { return V.getBitWidth() > 1; } + +public: + SizeOffsetAPInt() = default; + SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {} +}; /// Evaluate the size and offset of an object pointed to by a Value* /// statically. Fails if size or offset are not known at compile time. class ObjectSizeOffsetVisitor - : public InstVisitor { + : public InstVisitor { const DataLayout &DL; const TargetLibraryInfo *TLI; ObjectSizeOpts Options; unsigned IntTyBits; APInt Zero; - SmallDenseMap SeenInsts; + SmallDenseMap SeenInsts; unsigned InstructionsVisited; APInt align(APInt Size, MaybeAlign Align); - SizeOffsetType unknown() { - return std::make_pair(APInt(), APInt()); - } + static SizeOffsetAPInt unknown() { return SizeOffsetAPInt(); } public: ObjectSizeOffsetVisitor(const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context, ObjectSizeOpts Options = {}); - SizeOffsetType compute(Value *V); - - static bool knownSize(const SizeOffsetType &SizeOffset) { - return SizeOffset.first.getBitWidth() > 1; - } - - static bool knownOffset(const SizeOffsetType &SizeOffset) { - return SizeOffset.second.getBitWidth() > 1; - } - - static bool bothKnown(const SizeOffsetType &SizeOffset) { - return knownSize(SizeOffset) && knownOffset(SizeOffset); - } + SizeOffsetAPInt compute(Value *V); // These are "private", except they can't actually be made private. Only // compute() should be used by external users. - SizeOffsetType visitAllocaInst(AllocaInst &I); - SizeOffsetType visitArgument(Argument &A); - SizeOffsetType visitCallBase(CallBase &CB); - SizeOffsetType visitConstantPointerNull(ConstantPointerNull&); - SizeOffsetType visitExtractElementInst(ExtractElementInst &I); - SizeOffsetType visitExtractValueInst(ExtractValueInst &I); - SizeOffsetType visitGlobalAlias(GlobalAlias &GA); - SizeOffsetType visitGlobalVariable(GlobalVariable &GV); - SizeOffsetType visitIntToPtrInst(IntToPtrInst&); - SizeOffsetType visitLoadInst(LoadInst &I); - SizeOffsetType visitPHINode(PHINode&); - SizeOffsetType visitSelectInst(SelectInst &I); - SizeOffsetType visitUndefValue(UndefValue&); - SizeOffsetType visitInstruction(Instruction &I); + SizeOffsetAPInt visitAllocaInst(AllocaInst &I); + SizeOffsetAPInt visitArgument(Argument &A); + SizeOffsetAPInt visitCallBase(CallBase &CB); + SizeOffsetAPInt visitConstantPointerNull(ConstantPointerNull &); + SizeOffsetAPInt visitExtractElementInst(ExtractElementInst &I); + SizeOffsetAPInt visitExtractValueInst(ExtractValueInst &I); + SizeOffsetAPInt visitGlobalAlias(GlobalAlias &GA); + SizeOffsetAPInt visitGlobalVariable(GlobalVariable &GV); + SizeOffsetAPInt visitIntToPtrInst(IntToPtrInst &); + SizeOffsetAPInt visitLoadInst(LoadInst &I); + SizeOffsetAPInt visitPHINode(PHINode &); + SizeOffsetAPInt visitSelectInst(SelectInst &I); + SizeOffsetAPInt visitUndefValue(UndefValue &); + SizeOffsetAPInt visitInstruction(Instruction &I); private: - SizeOffsetType findLoadSizeOffset( + SizeOffsetAPInt findLoadSizeOffset( LoadInst &LoadFrom, BasicBlock &BB, BasicBlock::iterator From, - SmallDenseMap &VisitedBlocks, + SmallDenseMap &VisitedBlocks, unsigned &ScannedInstCount); - SizeOffsetType combineSizeOffset(SizeOffsetType LHS, SizeOffsetType RHS); - SizeOffsetType computeImpl(Value *V); - SizeOffsetType computeValue(Value *V); + SizeOffsetAPInt combineSizeOffset(SizeOffsetAPInt LHS, SizeOffsetAPInt RHS); + SizeOffsetAPInt computeImpl(Value *V); + SizeOffsetAPInt computeValue(Value *V); bool CheckedZextOrTrunc(APInt &I); }; -using SizeOffsetEvalType = std::pair; +/// SizeOffsetValue - Used by \p ObjectSizeOffsetEvaluator, which works with +/// \p Values. +class SizeOffsetWeakTrackingVH; +class SizeOffsetValue : public SizeOffsetType { + friend class SizeOffsetType; + static bool known(Value *V) { return V != nullptr; } + +public: + SizeOffsetValue() : SizeOffsetType(nullptr, nullptr) {} + SizeOffsetValue(Value *Size, Value *Offset) : SizeOffsetType(Size, Offset) {} + SizeOffsetValue(const SizeOffsetWeakTrackingVH &SOT); +}; + +/// SizeOffsetWeakTrackingVH - Used by \p ObjectSizeOffsetEvaluator in a +/// \p DenseMap. +class SizeOffsetWeakTrackingVH + : public SizeOffsetType { + friend class SizeOffsetType; + static bool known(WeakTrackingVH V) { return V.pointsToAliveValue(); } + +public: + SizeOffsetWeakTrackingVH() : SizeOffsetType(nullptr, nullptr) {} + SizeOffsetWeakTrackingVH(Value *Size, Value *Offset) + : SizeOffsetType(Size, Offset) {} + SizeOffsetWeakTrackingVH(const SizeOffsetValue &SOV) + : SizeOffsetType(SOV.Size, SOV.Offset) {} +}; /// Evaluate the size and offset of an object pointed to by a Value*. /// May create code to compute the result at run-time. class ObjectSizeOffsetEvaluator - : public InstVisitor { + : public InstVisitor { using BuilderTy = IRBuilder; - using WeakEvalType = std::pair; + using WeakEvalType = SizeOffsetWeakTrackingVH; using CacheMapTy = DenseMap; using PtrSetTy = SmallPtrSet; @@ -275,45 +320,27 @@ class ObjectSizeOffsetEvaluator ObjectSizeOpts EvalOpts; SmallPtrSet InsertedInstructions; - SizeOffsetEvalType compute_(Value *V); + SizeOffsetValue compute_(Value *V); public: - static SizeOffsetEvalType unknown() { - return std::make_pair(nullptr, nullptr); - } - ObjectSizeOffsetEvaluator(const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context, ObjectSizeOpts EvalOpts = {}); - SizeOffsetEvalType compute(Value *V); - - bool knownSize(SizeOffsetEvalType SizeOffset) { - return SizeOffset.first; - } - - bool knownOffset(SizeOffsetEvalType SizeOffset) { - return SizeOffset.second; - } - - bool anyKnown(SizeOffsetEvalType SizeOffset) { - return knownSize(SizeOffset) || knownOffset(SizeOffset); - } + static SizeOffsetValue unknown() { return SizeOffsetValue(); } - bool bothKnown(SizeOffsetEvalType SizeOffset) { - return knownSize(SizeOffset) && knownOffset(SizeOffset); - } + SizeOffsetValue compute(Value *V); // The individual instruction visitors should be treated as private. - SizeOffsetEvalType visitAllocaInst(AllocaInst &I); - SizeOffsetEvalType visitCallBase(CallBase &CB); - SizeOffsetEvalType visitExtractElementInst(ExtractElementInst &I); - SizeOffsetEvalType visitExtractValueInst(ExtractValueInst &I); - SizeOffsetEvalType visitGEPOperator(GEPOperator &GEP); - SizeOffsetEvalType visitIntToPtrInst(IntToPtrInst&); - SizeOffsetEvalType visitLoadInst(LoadInst &I); - SizeOffsetEvalType visitPHINode(PHINode &PHI); - SizeOffsetEvalType visitSelectInst(SelectInst &I); - SizeOffsetEvalType visitInstruction(Instruction &I); + SizeOffsetValue visitAllocaInst(AllocaInst &I); + SizeOffsetValue visitCallBase(CallBase &CB); + SizeOffsetValue visitExtractElementInst(ExtractElementInst &I); + SizeOffsetValue visitExtractValueInst(ExtractValueInst &I); + SizeOffsetValue visitGEPOperator(GEPOperator &GEP); + SizeOffsetValue visitIntToPtrInst(IntToPtrInst &); + SizeOffsetValue visitLoadInst(LoadInst &I); + SizeOffsetValue visitPHINode(PHINode &PHI); + SizeOffsetValue visitSelectInst(SelectInst &I); + SizeOffsetValue visitInstruction(Instruction &I); }; } // end namespace llvm diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp index 9e6811f3bf88..46a7a921d86d 100644 --- a/llvm/lib/Analysis/MemoryBuiltins.cpp +++ b/llvm/lib/Analysis/MemoryBuiltins.cpp @@ -577,10 +577,12 @@ Value *llvm::getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI) { //===----------------------------------------------------------------------===// // Utility functions to compute size of objects. // -static APInt getSizeWithOverflow(const SizeOffsetType &Data) { - if (Data.second.isNegative() || Data.first.ult(Data.second)) - return APInt(Data.first.getBitWidth(), 0); - return Data.first - Data.second; +static APInt getSizeWithOverflow(const SizeOffsetAPInt &Data) { + APInt Size = Data.Size; + APInt Offset = Data.Offset; + if (Offset.isNegative() || Size.ult(Offset)) + return APInt(Size.getBitWidth(), 0); + return Size - Offset; } /// Compute the size of the object pointed by Ptr. Returns true and the @@ -590,8 +592,8 @@ static APInt getSizeWithOverflow(const SizeOffsetType &Data) { bool llvm::getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL, const TargetLibraryInfo *TLI, ObjectSizeOpts Opts) { ObjectSizeOffsetVisitor Visitor(DL, TLI, Ptr->getContext(), Opts); - SizeOffsetType Data = Visitor.compute(const_cast(Ptr)); - if (!Visitor.bothKnown(Data)) + SizeOffsetAPInt Data = Visitor.compute(const_cast(Ptr)); + if (!Data.bothKnown()) return false; Size = getSizeWithOverflow(Data).getZExtValue(); @@ -640,8 +642,7 @@ Value *llvm::lowerObjectSizeCall( } else { LLVMContext &Ctx = ObjectSize->getFunction()->getContext(); ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, EvalOptions); - SizeOffsetEvalType SizeOffsetPair = - Eval.compute(ObjectSize->getArgOperand(0)); + SizeOffsetValue SizeOffsetPair = Eval.compute(ObjectSize->getArgOperand(0)); if (SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown()) { IRBuilder Builder( @@ -651,19 +652,19 @@ Value *llvm::lowerObjectSizeCall( })); Builder.SetInsertPoint(ObjectSize); + Value *Size = SizeOffsetPair.Size; + Value *Offset = SizeOffsetPair.Offset; + // If we've outside the end of the object, then we can always access // exactly 0 bytes. - Value *ResultSize = - Builder.CreateSub(SizeOffsetPair.first, SizeOffsetPair.second); - Value *UseZero = - Builder.CreateICmpULT(SizeOffsetPair.first, SizeOffsetPair.second); + Value *ResultSize = Builder.CreateSub(Size, Offset); + Value *UseZero = Builder.CreateICmpULT(Size, Offset); ResultSize = Builder.CreateZExtOrTrunc(ResultSize, ResultType); Value *Ret = Builder.CreateSelect( UseZero, ConstantInt::get(ResultType, 0), ResultSize); // The non-constant size expression cannot evaluate to -1. - if (!isa(SizeOffsetPair.first) || - !isa(SizeOffsetPair.second)) + if (!isa(Size) || !isa(Offset)) Builder.CreateAssumption( Builder.CreateICmpNE(Ret, ConstantInt::get(ResultType, -1))); @@ -697,12 +698,12 @@ ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const DataLayout &DL, // a different address space. } -SizeOffsetType ObjectSizeOffsetVisitor::compute(Value *V) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::compute(Value *V) { InstructionsVisited = 0; return computeImpl(V); } -SizeOffsetType ObjectSizeOffsetVisitor::computeImpl(Value *V) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::computeImpl(Value *V) { unsigned InitialIntTyBits = DL.getIndexTypeSizeInBits(V->getType()); // Stripping pointer casts can strip address space casts which can change the @@ -719,7 +720,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::computeImpl(Value *V) { IntTyBits = DL.getIndexTypeSizeInBits(V->getType()); Zero = APInt::getZero(IntTyBits); - SizeOffsetType SOT = computeValue(V); + SizeOffsetAPInt SOT = computeValue(V); bool IndexTypeSizeChanged = InitialIntTyBits != IntTyBits; if (!IndexTypeSizeChanged && Offset.isZero()) @@ -729,27 +730,28 @@ SizeOffsetType ObjectSizeOffsetVisitor::computeImpl(Value *V) { // accumulated some constant offset (or both). Readjust the bit width to match // the argument index type size and apply the offset, as required. if (IndexTypeSizeChanged) { - if (knownSize(SOT) && !::CheckedZextOrTrunc(SOT.first, InitialIntTyBits)) - SOT.first = APInt(); - if (knownOffset(SOT) && !::CheckedZextOrTrunc(SOT.second, InitialIntTyBits)) - SOT.second = APInt(); + if (SOT.knownSize() && !::CheckedZextOrTrunc(SOT.Size, InitialIntTyBits)) + SOT.Size = APInt(); + if (SOT.knownOffset() && + !::CheckedZextOrTrunc(SOT.Offset, InitialIntTyBits)) + SOT.Offset = APInt(); } // If the computed offset is "unknown" we cannot add the stripped offset. - return {SOT.first, - SOT.second.getBitWidth() > 1 ? SOT.second + Offset : SOT.second}; + return {SOT.Size, + SOT.Offset.getBitWidth() > 1 ? SOT.Offset + Offset : SOT.Offset}; } -SizeOffsetType ObjectSizeOffsetVisitor::computeValue(Value *V) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::computeValue(Value *V) { if (Instruction *I = dyn_cast(V)) { // If we have already seen this instruction, bail out. Cycles can happen in // unreachable code after constant propagation. - auto P = SeenInsts.try_emplace(I, unknown()); + auto P = SeenInsts.try_emplace(I, ObjectSizeOffsetVisitor::unknown()); if (!P.second) return P.first->second; ++InstructionsVisited; if (InstructionsVisited > ObjectSizeOffsetVisitorMaxVisitInstructions) - return unknown(); - SizeOffsetType Res = visit(*I); + return ObjectSizeOffsetVisitor::unknown(); + SizeOffsetAPInt Res = visit(*I); // Cache the result for later visits. If we happened to visit this during // the above recursion, we would consider it unknown until now. SeenInsts[I] = Res; @@ -768,55 +770,55 @@ SizeOffsetType ObjectSizeOffsetVisitor::computeValue(Value *V) { LLVM_DEBUG(dbgs() << "ObjectSizeOffsetVisitor::compute() unhandled value: " << *V << '\n'); - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); } bool ObjectSizeOffsetVisitor::CheckedZextOrTrunc(APInt &I) { return ::CheckedZextOrTrunc(I, IntTyBits); } -SizeOffsetType ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) { TypeSize ElemSize = DL.getTypeAllocSize(I.getAllocatedType()); if (ElemSize.isScalable() && Options.EvalMode != ObjectSizeOpts::Mode::Min) - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); APInt Size(IntTyBits, ElemSize.getKnownMinValue()); if (!I.isArrayAllocation()) - return std::make_pair(align(Size, I.getAlign()), Zero); + return SizeOffsetAPInt(align(Size, I.getAlign()), Zero); Value *ArraySize = I.getArraySize(); if (const ConstantInt *C = dyn_cast(ArraySize)) { APInt NumElems = C->getValue(); if (!CheckedZextOrTrunc(NumElems)) - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); bool Overflow; Size = Size.umul_ov(NumElems, Overflow); - return Overflow ? unknown() - : std::make_pair(align(Size, I.getAlign()), Zero); + return Overflow ? ObjectSizeOffsetVisitor::unknown() + : SizeOffsetAPInt(align(Size, I.getAlign()), Zero); } - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); } -SizeOffsetType ObjectSizeOffsetVisitor::visitArgument(Argument &A) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitArgument(Argument &A) { Type *MemoryTy = A.getPointeeInMemoryValueType(); // No interprocedural analysis is done at the moment. if (!MemoryTy|| !MemoryTy->isSized()) { ++ObjectVisitorArgument; - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); } APInt Size(IntTyBits, DL.getTypeAllocSize(MemoryTy)); - return std::make_pair(align(Size, A.getParamAlign()), Zero); + return SizeOffsetAPInt(align(Size, A.getParamAlign()), Zero); } -SizeOffsetType ObjectSizeOffsetVisitor::visitCallBase(CallBase &CB) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitCallBase(CallBase &CB) { if (std::optional Size = getAllocSize(&CB, TLI)) - return std::make_pair(*Size, Zero); - return unknown(); + return SizeOffsetAPInt(*Size, Zero); + return ObjectSizeOffsetVisitor::unknown(); } -SizeOffsetType -ObjectSizeOffsetVisitor::visitConstantPointerNull(ConstantPointerNull& CPN) { +SizeOffsetAPInt +ObjectSizeOffsetVisitor::visitConstantPointerNull(ConstantPointerNull &CPN) { // If null is unknown, there's nothing we can do. Additionally, non-zero // address spaces can make use of null, so we don't presume to know anything // about that. @@ -825,45 +827,46 @@ ObjectSizeOffsetVisitor::visitConstantPointerNull(ConstantPointerNull& CPN) { // them on the floor, but it's unclear what we should do when a NULL from // addrspace(1) gets casted to addrspace(0) (or vice-versa). if (Options.NullIsUnknownSize || CPN.getType()->getAddressSpace()) - return unknown(); - return std::make_pair(Zero, Zero); + return ObjectSizeOffsetVisitor::unknown(); + return SizeOffsetAPInt(Zero, Zero); } -SizeOffsetType -ObjectSizeOffsetVisitor::visitExtractElementInst(ExtractElementInst&) { - return unknown(); +SizeOffsetAPInt +ObjectSizeOffsetVisitor::visitExtractElementInst(ExtractElementInst &) { + return ObjectSizeOffsetVisitor::unknown(); } -SizeOffsetType -ObjectSizeOffsetVisitor::visitExtractValueInst(ExtractValueInst&) { +SizeOffsetAPInt +ObjectSizeOffsetVisitor::visitExtractValueInst(ExtractValueInst &) { // Easy cases were already folded by previous passes. - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); } -SizeOffsetType ObjectSizeOffsetVisitor::visitGlobalAlias(GlobalAlias &GA) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitGlobalAlias(GlobalAlias &GA) { if (GA.isInterposable()) - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); return computeImpl(GA.getAliasee()); } -SizeOffsetType ObjectSizeOffsetVisitor::visitGlobalVariable(GlobalVariable &GV){ +SizeOffsetAPInt +ObjectSizeOffsetVisitor::visitGlobalVariable(GlobalVariable &GV) { if (!GV.getValueType()->isSized() || GV.hasExternalWeakLinkage() || ((!GV.hasInitializer() || GV.isInterposable()) && Options.EvalMode != ObjectSizeOpts::Mode::Min)) - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); APInt Size(IntTyBits, DL.getTypeAllocSize(GV.getValueType())); - return std::make_pair(align(Size, GV.getAlign()), Zero); + return SizeOffsetAPInt(align(Size, GV.getAlign()), Zero); } -SizeOffsetType ObjectSizeOffsetVisitor::visitIntToPtrInst(IntToPtrInst&) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitIntToPtrInst(IntToPtrInst &) { // clueless - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); } -SizeOffsetType ObjectSizeOffsetVisitor::findLoadSizeOffset( +SizeOffsetAPInt ObjectSizeOffsetVisitor::findLoadSizeOffset( LoadInst &Load, BasicBlock &BB, BasicBlock::iterator From, - SmallDenseMap &VisitedBlocks, + SmallDenseMap &VisitedBlocks, unsigned &ScannedInstCount) { constexpr unsigned MaxInstsToScan = 128; @@ -871,10 +874,10 @@ SizeOffsetType ObjectSizeOffsetVisitor::findLoadSizeOffset( if (Where != VisitedBlocks.end()) return Where->second; - auto Unknown = [this, &BB, &VisitedBlocks]() { - return VisitedBlocks[&BB] = unknown(); + auto Unknown = [&BB, &VisitedBlocks]() { + return VisitedBlocks[&BB] = ObjectSizeOffsetVisitor::unknown(); }; - auto Known = [&BB, &VisitedBlocks](SizeOffsetType SO) { + auto Known = [&BB, &VisitedBlocks](SizeOffsetAPInt SO) { return VisitedBlocks[&BB] = SO; }; @@ -951,46 +954,47 @@ SizeOffsetType ObjectSizeOffsetVisitor::findLoadSizeOffset( return Unknown(); } while (From-- != BB.begin()); - SmallVector PredecessorSizeOffsets; + SmallVector PredecessorSizeOffsets; for (auto *PredBB : predecessors(&BB)) { PredecessorSizeOffsets.push_back(findLoadSizeOffset( Load, *PredBB, BasicBlock::iterator(PredBB->getTerminator()), VisitedBlocks, ScannedInstCount)); - if (!bothKnown(PredecessorSizeOffsets.back())) + if (!PredecessorSizeOffsets.back().bothKnown()) return Unknown(); } if (PredecessorSizeOffsets.empty()) return Unknown(); - return Known(std::accumulate(PredecessorSizeOffsets.begin() + 1, - PredecessorSizeOffsets.end(), - PredecessorSizeOffsets.front(), - [this](SizeOffsetType LHS, SizeOffsetType RHS) { - return combineSizeOffset(LHS, RHS); - })); + return Known(std::accumulate( + PredecessorSizeOffsets.begin() + 1, PredecessorSizeOffsets.end(), + PredecessorSizeOffsets.front(), + [this](SizeOffsetAPInt LHS, SizeOffsetAPInt RHS) { + return combineSizeOffset(LHS, RHS); + })); } -SizeOffsetType ObjectSizeOffsetVisitor::visitLoadInst(LoadInst &LI) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitLoadInst(LoadInst &LI) { if (!Options.AA) { ++ObjectVisitorLoad; - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); } - SmallDenseMap VisitedBlocks; + SmallDenseMap VisitedBlocks; unsigned ScannedInstCount = 0; - SizeOffsetType SO = + SizeOffsetAPInt SO = findLoadSizeOffset(LI, *LI.getParent(), BasicBlock::iterator(LI), VisitedBlocks, ScannedInstCount); - if (!bothKnown(SO)) + if (!SO.bothKnown()) ++ObjectVisitorLoad; return SO; } -SizeOffsetType ObjectSizeOffsetVisitor::combineSizeOffset(SizeOffsetType LHS, - SizeOffsetType RHS) { - if (!bothKnown(LHS) || !bothKnown(RHS)) - return unknown(); +SizeOffsetAPInt +ObjectSizeOffsetVisitor::combineSizeOffset(SizeOffsetAPInt LHS, + SizeOffsetAPInt RHS) { + if (!LHS.bothKnown() || !RHS.bothKnown()) + return ObjectSizeOffsetVisitor::unknown(); switch (Options.EvalMode) { case ObjectSizeOpts::Mode::Min: @@ -998,40 +1002,45 @@ SizeOffsetType ObjectSizeOffsetVisitor::combineSizeOffset(SizeOffsetType LHS, case ObjectSizeOpts::Mode::Max: return (getSizeWithOverflow(LHS).sgt(getSizeWithOverflow(RHS))) ? LHS : RHS; case ObjectSizeOpts::Mode::ExactSizeFromOffset: - return (getSizeWithOverflow(LHS).eq(getSizeWithOverflow(RHS))) ? LHS - : unknown(); + return (getSizeWithOverflow(LHS).eq(getSizeWithOverflow(RHS))) + ? LHS + : ObjectSizeOffsetVisitor::unknown(); case ObjectSizeOpts::Mode::ExactUnderlyingSizeAndOffset: - return LHS == RHS ? LHS : unknown(); + return LHS == RHS ? LHS : ObjectSizeOffsetVisitor::unknown(); } llvm_unreachable("missing an eval mode"); } -SizeOffsetType ObjectSizeOffsetVisitor::visitPHINode(PHINode &PN) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitPHINode(PHINode &PN) { if (PN.getNumIncomingValues() == 0) - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); auto IncomingValues = PN.incoming_values(); return std::accumulate(IncomingValues.begin() + 1, IncomingValues.end(), computeImpl(*IncomingValues.begin()), - [this](SizeOffsetType LHS, Value *VRHS) { + [this](SizeOffsetAPInt LHS, Value *VRHS) { return combineSizeOffset(LHS, computeImpl(VRHS)); }); } -SizeOffsetType ObjectSizeOffsetVisitor::visitSelectInst(SelectInst &I) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitSelectInst(SelectInst &I) { return combineSizeOffset(computeImpl(I.getTrueValue()), computeImpl(I.getFalseValue())); } -SizeOffsetType ObjectSizeOffsetVisitor::visitUndefValue(UndefValue&) { - return std::make_pair(Zero, Zero); +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitUndefValue(UndefValue &) { + return SizeOffsetAPInt(Zero, Zero); } -SizeOffsetType ObjectSizeOffsetVisitor::visitInstruction(Instruction &I) { +SizeOffsetAPInt ObjectSizeOffsetVisitor::visitInstruction(Instruction &I) { LLVM_DEBUG(dbgs() << "ObjectSizeOffsetVisitor unknown instruction:" << I << '\n'); - return unknown(); + return ObjectSizeOffsetVisitor::unknown(); } +// Just set these right here... +SizeOffsetValue::SizeOffsetValue(const SizeOffsetWeakTrackingVH &SOT) + : SizeOffsetType(SOT.Size, SOT.Offset) {} + ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator( const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context, ObjectSizeOpts EvalOpts) @@ -1044,21 +1053,21 @@ ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator( // be different for later objects. } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute(Value *V) { +SizeOffsetValue ObjectSizeOffsetEvaluator::compute(Value *V) { // XXX - Are vectors of pointers possible here? IntTy = cast(DL.getIndexType(V->getType())); Zero = ConstantInt::get(IntTy, 0); - SizeOffsetEvalType Result = compute_(V); + SizeOffsetValue Result = compute_(V); - if (!bothKnown(Result)) { + if (!Result.bothKnown()) { // Erase everything that was computed in this iteration from the cache, so // that no dangling references are left behind. We could be a bit smarter if // we kept a dependency graph. It's probably not worth the complexity. for (const Value *SeenVal : SeenVals) { CacheMapTy::iterator CacheIt = CacheMap.find(SeenVal); // non-computable results can be safely cached - if (CacheIt != CacheMap.end() && anyKnown(CacheIt->second)) + if (CacheIt != CacheMap.end() && CacheIt->second.anyKnown()) CacheMap.erase(CacheIt); } @@ -1074,12 +1083,12 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute(Value *V) { return Result; } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute_(Value *V) { +SizeOffsetValue ObjectSizeOffsetEvaluator::compute_(Value *V) { ObjectSizeOffsetVisitor Visitor(DL, TLI, Context, EvalOpts); - SizeOffsetType Const = Visitor.compute(V); - if (Visitor.bothKnown(Const)) - return std::make_pair(ConstantInt::get(Context, Const.first), - ConstantInt::get(Context, Const.second)); + SizeOffsetAPInt Const = Visitor.compute(V); + if (Const.bothKnown()) + return SizeOffsetValue(ConstantInt::get(Context, Const.Size), + ConstantInt::get(Context, Const.Offset)); V = V->stripPointerCasts(); @@ -1095,13 +1104,13 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute_(Value *V) { Builder.SetInsertPoint(I); // Now compute the size and offset. - SizeOffsetEvalType Result; + SizeOffsetValue Result; // Record the pointers that were handled in this run, so that they can be // cleaned later if something fails. We also use this set to break cycles that // can occur in dead code. if (!SeenVals.insert(V).second) { - Result = unknown(); + Result = ObjectSizeOffsetEvaluator::unknown(); } else if (GEPOperator *GEP = dyn_cast(V)) { Result = visitGEPOperator(*GEP); } else if (Instruction *I = dyn_cast(V)) { @@ -1112,22 +1121,22 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute_(Value *V) { isa(V) || isa(V)) { // Ignore values where we cannot do more than ObjectSizeVisitor. - Result = unknown(); + Result = ObjectSizeOffsetEvaluator::unknown(); } else { LLVM_DEBUG( dbgs() << "ObjectSizeOffsetEvaluator::compute() unhandled value: " << *V << '\n'); - Result = unknown(); + Result = ObjectSizeOffsetEvaluator::unknown(); } // Don't reuse CacheIt since it may be invalid at this point. - CacheMap[V] = Result; + CacheMap[V] = SizeOffsetWeakTrackingVH(Result); return Result; } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitAllocaInst(AllocaInst &I) { +SizeOffsetValue ObjectSizeOffsetEvaluator::visitAllocaInst(AllocaInst &I) { if (!I.getAllocatedType()->isSized()) - return unknown(); + return ObjectSizeOffsetEvaluator::unknown(); // must be a VLA assert(I.isArrayAllocation()); @@ -1143,86 +1152,85 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitAllocaInst(AllocaInst &I) { Value *Size = ConstantInt::get(ArraySize->getType(), DL.getTypeAllocSize(I.getAllocatedType())); Size = Builder.CreateMul(Size, ArraySize); - return std::make_pair(Size, Zero); + return SizeOffsetValue(Size, Zero); } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitCallBase(CallBase &CB) { +SizeOffsetValue ObjectSizeOffsetEvaluator::visitCallBase(CallBase &CB) { std::optional FnData = getAllocationSize(&CB, TLI); if (!FnData) - return unknown(); + return ObjectSizeOffsetEvaluator::unknown(); // Handle strdup-like functions separately. if (FnData->AllocTy == StrDupLike) { // TODO: implement evaluation of strdup/strndup - return unknown(); + return ObjectSizeOffsetEvaluator::unknown(); } Value *FirstArg = CB.getArgOperand(FnData->FstParam); FirstArg = Builder.CreateZExtOrTrunc(FirstArg, IntTy); if (FnData->SndParam < 0) - return std::make_pair(FirstArg, Zero); + return SizeOffsetValue(FirstArg, Zero); Value *SecondArg = CB.getArgOperand(FnData->SndParam); SecondArg = Builder.CreateZExtOrTrunc(SecondArg, IntTy); Value *Size = Builder.CreateMul(FirstArg, SecondArg); - return std::make_pair(Size, Zero); + return SizeOffsetValue(Size, Zero); } -SizeOffsetEvalType -ObjectSizeOffsetEvaluator::visitExtractElementInst(ExtractElementInst&) { - return unknown(); +SizeOffsetValue +ObjectSizeOffsetEvaluator::visitExtractElementInst(ExtractElementInst &) { + return ObjectSizeOffsetEvaluator::unknown(); } -SizeOffsetEvalType -ObjectSizeOffsetEvaluator::visitExtractValueInst(ExtractValueInst&) { - return unknown(); +SizeOffsetValue +ObjectSizeOffsetEvaluator::visitExtractValueInst(ExtractValueInst &) { + return ObjectSizeOffsetEvaluator::unknown(); } -SizeOffsetEvalType -ObjectSizeOffsetEvaluator::visitGEPOperator(GEPOperator &GEP) { - SizeOffsetEvalType PtrData = compute_(GEP.getPointerOperand()); - if (!bothKnown(PtrData)) - return unknown(); +SizeOffsetValue ObjectSizeOffsetEvaluator::visitGEPOperator(GEPOperator &GEP) { + SizeOffsetValue PtrData = compute_(GEP.getPointerOperand()); + if (!PtrData.bothKnown()) + return ObjectSizeOffsetEvaluator::unknown(); Value *Offset = emitGEPOffset(&Builder, DL, &GEP, /*NoAssumptions=*/true); - Offset = Builder.CreateAdd(PtrData.second, Offset); - return std::make_pair(PtrData.first, Offset); + Offset = Builder.CreateAdd(PtrData.Offset, Offset); + return SizeOffsetValue(PtrData.Size, Offset); } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitIntToPtrInst(IntToPtrInst&) { +SizeOffsetValue ObjectSizeOffsetEvaluator::visitIntToPtrInst(IntToPtrInst &) { // clueless - return unknown(); + return ObjectSizeOffsetEvaluator::unknown(); } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitLoadInst(LoadInst &LI) { - return unknown(); +SizeOffsetValue ObjectSizeOffsetEvaluator::visitLoadInst(LoadInst &LI) { + return ObjectSizeOffsetEvaluator::unknown(); } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitPHINode(PHINode &PHI) { +SizeOffsetValue ObjectSizeOffsetEvaluator::visitPHINode(PHINode &PHI) { // Create 2 PHIs: one for size and another for offset. PHINode *SizePHI = Builder.CreatePHI(IntTy, PHI.getNumIncomingValues()); PHINode *OffsetPHI = Builder.CreatePHI(IntTy, PHI.getNumIncomingValues()); // Insert right away in the cache to handle recursive PHIs. - CacheMap[&PHI] = std::make_pair(SizePHI, OffsetPHI); + CacheMap[&PHI] = SizeOffsetWeakTrackingVH(SizePHI, OffsetPHI); // Compute offset/size for each PHI incoming pointer. for (unsigned i = 0, e = PHI.getNumIncomingValues(); i != e; ++i) { BasicBlock *IncomingBlock = PHI.getIncomingBlock(i); Builder.SetInsertPoint(IncomingBlock, IncomingBlock->getFirstInsertionPt()); - SizeOffsetEvalType EdgeData = compute_(PHI.getIncomingValue(i)); + SizeOffsetValue EdgeData = compute_(PHI.getIncomingValue(i)); - if (!bothKnown(EdgeData)) { + if (!EdgeData.bothKnown()) { OffsetPHI->replaceAllUsesWith(PoisonValue::get(IntTy)); OffsetPHI->eraseFromParent(); InsertedInstructions.erase(OffsetPHI); SizePHI->replaceAllUsesWith(PoisonValue::get(IntTy)); SizePHI->eraseFromParent(); InsertedInstructions.erase(SizePHI); - return unknown(); + return ObjectSizeOffsetEvaluator::unknown(); } - SizePHI->addIncoming(EdgeData.first, IncomingBlock); - OffsetPHI->addIncoming(EdgeData.second, IncomingBlock); + SizePHI->addIncoming(EdgeData.Size, IncomingBlock); + OffsetPHI->addIncoming(EdgeData.Offset, IncomingBlock); } Value *Size = SizePHI, *Offset = OffsetPHI; @@ -1238,27 +1246,27 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitPHINode(PHINode &PHI) { OffsetPHI->eraseFromParent(); InsertedInstructions.erase(OffsetPHI); } - return std::make_pair(Size, Offset); + return SizeOffsetValue(Size, Offset); } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitSelectInst(SelectInst &I) { - SizeOffsetEvalType TrueSide = compute_(I.getTrueValue()); - SizeOffsetEvalType FalseSide = compute_(I.getFalseValue()); +SizeOffsetValue ObjectSizeOffsetEvaluator::visitSelectInst(SelectInst &I) { + SizeOffsetValue TrueSide = compute_(I.getTrueValue()); + SizeOffsetValue FalseSide = compute_(I.getFalseValue()); - if (!bothKnown(TrueSide) || !bothKnown(FalseSide)) - return unknown(); + if (!TrueSide.bothKnown() || !FalseSide.bothKnown()) + return ObjectSizeOffsetEvaluator::unknown(); if (TrueSide == FalseSide) return TrueSide; - Value *Size = Builder.CreateSelect(I.getCondition(), TrueSide.first, - FalseSide.first); - Value *Offset = Builder.CreateSelect(I.getCondition(), TrueSide.second, - FalseSide.second); - return std::make_pair(Size, Offset); + Value *Size = + Builder.CreateSelect(I.getCondition(), TrueSide.Size, FalseSide.Size); + Value *Offset = + Builder.CreateSelect(I.getCondition(), TrueSide.Offset, FalseSide.Offset); + return SizeOffsetValue(Size, Offset); } -SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitInstruction(Instruction &I) { +SizeOffsetValue ObjectSizeOffsetEvaluator::visitInstruction(Instruction &I) { LLVM_DEBUG(dbgs() << "ObjectSizeOffsetEvaluator unknown instruction:" << I << '\n'); - return unknown(); + return ObjectSizeOffsetEvaluator::unknown(); } diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index b2618e35b085..cc5a4ee8c2bd 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -6725,10 +6725,10 @@ struct AAHeapToStackFunction final : public AAHeapToStack { LLVMContext &Ctx = AI.CB->getContext(); ObjectSizeOpts Opts; ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts); - SizeOffsetEvalType SizeOffsetPair = Eval.compute(AI.CB); + SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB); assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() && - cast(SizeOffsetPair.second)->isZero()); - Size = SizeOffsetPair.first; + cast(SizeOffsetPair.Offset)->isZero()); + Size = SizeOffsetPair.Size; } Instruction *IP = diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index d4f5bf8c3935..e3deafa49bd9 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -3665,10 +3665,14 @@ bool AddressSanitizer::isSafeAccess(ObjectSizeOffsetVisitor &ObjSizeVis, // TODO: We can use vscale_range to convert a scalable value to an // upper bound on the access size. return false; - SizeOffsetType SizeOffset = ObjSizeVis.compute(Addr); - if (!ObjSizeVis.bothKnown(SizeOffset)) return false; - uint64_t Size = SizeOffset.first.getZExtValue(); - int64_t Offset = SizeOffset.second.getSExtValue(); + + SizeOffsetAPInt SizeOffset = ObjSizeVis.compute(Addr); + if (!SizeOffset.bothKnown()) + return false; + + uint64_t Size = SizeOffset.Size.getZExtValue(); + int64_t Offset = SizeOffset.Offset.getSExtValue(); + // Three checks are required to ensure safety: // . Offset >= 0 (since the offset is given from the base ptr) // . Size >= Offset (unsigned) diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp index ee5b81960417..cfa8ae26c625 100644 --- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp +++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp @@ -61,15 +61,15 @@ static Value *getBoundsCheckCond(Value *Ptr, Value *InstVal, LLVM_DEBUG(dbgs() << "Instrument " << *Ptr << " for " << Twine(NeededSize) << " bytes\n"); - SizeOffsetEvalType SizeOffset = ObjSizeEval.compute(Ptr); + SizeOffsetValue SizeOffset = ObjSizeEval.compute(Ptr); - if (!ObjSizeEval.bothKnown(SizeOffset)) { + if (!SizeOffset.bothKnown()) { ++ChecksUnable; return nullptr; } - Value *Size = SizeOffset.first; - Value *Offset = SizeOffset.second; + Value *Size = SizeOffset.Size; + Value *Offset = SizeOffset.Offset; ConstantInt *SizeCI = dyn_cast(Size); Type *IndexTy = DL.getIndexType(Ptr->getType()); -- GitLab From 9a95b2481e5ee0708aa8604e0c04b5bd1041de3a Mon Sep 17 00:00:00 2001 From: Valentin Clement Date: Fri, 5 Jan 2024 18:11:23 -0800 Subject: [PATCH 177/728] [flang][openacc][NFC] Remove obsolete FIR testing in acc-reduction.f90 test Most tests were updated in #73054. This is the last one for OpenACC lowering --- flang/test/Lower/OpenACC/acc-reduction.f90 | 539 ++++++++++----------- 1 file changed, 258 insertions(+), 281 deletions(-) diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90 index dcfa77c9f97d..fe04c3114b7e 100644 --- a/flang/test/Lower/OpenACC/acc-reduction.f90 +++ b/flang/test/Lower/OpenACC/acc-reduction.f90 @@ -1,35 +1,34 @@ ! This test checks lowering of OpenACC reduction clause. -! RUN: bbc -fopenacc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes=CHECK,FIR -! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,HLFIR +! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK ! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_UxUxf32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index): -! HLFIR: %[[CST:.*]] = arith.constant -1.401300e-45 : f32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %arg1, %arg2 : (index, index) -> !fir.shape<2> -! HLFIR: %[[TEMP:.*]] = fir.alloca !fir.array, %arg1, %arg2 -! HLFIR: %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) -! HLFIR: hlfir.assign %[[CST]] to %[[DECL]]#0 : f32, !fir.box> -! HLFIR: acc.yield %[[DECL]]#0 : !fir.box> +! CHECK: %[[CST:.*]] = arith.constant -1.401300e-45 : f32 +! CHECK: %[[SHAPE:.*]] = fir.shape %arg1, %arg2 : (index, index) -> !fir.shape<2> +! CHECK: %[[TEMP:.*]] = fir.alloca !fir.array, %arg1, %arg2 +! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) +! CHECK: hlfir.assign %[[CST]] to %[[DECL]]#0 : f32, !fir.box> +! CHECK: acc.yield %[[DECL]]#0 : !fir.box> ! CHECK: } combiner { ! CHECK: ^bb0(%[[V1:.*]]: !fir.ref>, %[[V2:.*]]: !fir.ref>, %[[LB0:.*]]: index, %[[UB0:.*]]: index, %[[STEP0:.*]]: index, %[[LB1:.*]]: index, %[[UB1:.*]]: index, %[[STEP1:.*]]: index): -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> -! HLFIR: %[[DECL_V1:.*]]:2 = hlfir.declare %[[V1]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) -! HLFIR: %[[DECL_V2:.*]]:2 = hlfir.declare %[[V2]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) -! HLFIR: %[[DES_V1:.*]] = hlfir.designate %[[DECL_V1]]#0 (%arg2:%arg3:%arg4, %arg5:%arg6:%arg7) shape %10 : (!fir.box>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box> -! HLFIR: %[[DES_V2:.*]] = hlfir.designate %[[DECL_V2]]#0 (%arg2:%arg3:%arg4, %arg5:%arg6:%arg7) shape %10 : (!fir.box>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box> -! HLFIR: %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<2>) -> !hlfir.expr { -! HLFIR: ^bb0(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index): -! HLFIR: %[[D1:.*]] = hlfir.designate %13 (%[[ARG0]], %[[ARG1]]) : (!fir.box>, index, index) -> !fir.ref -! HLFIR: %[[D2:.*]] = hlfir.designate %14 (%[[ARG0]], %[[ARG1]]) : (!fir.box>, index, index) -> !fir.ref -! HLFIR: %[[LOAD1:.*]] = fir.load %[[D1]] : !fir.ref -! HLFIR: %[[LOAD2:.*]] = fir.load %[[D2]] : !fir.ref -! HLFIR: %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD1]], %[[LOAD2]] {{.*}} : f32 -! HLFIR: %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : f32 -! HLFIR: hlfir.yield_element %[[SELECT]] : f32 -! HLFIR: } -! HLFIR: hlfir.assign %[[ELEMENTAL]] to %[[DECL_V1]]#0 : !hlfir.expr, !fir.box> -! HLFIR: acc.yield %[[V1]] : !fir.ref> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> +! CHECK: %[[DECL_V1:.*]]:2 = hlfir.declare %[[V1]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) +! CHECK: %[[DECL_V2:.*]]:2 = hlfir.declare %[[V2]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) +! CHECK: %[[DES_V1:.*]] = hlfir.designate %[[DECL_V1]]#0 (%arg2:%arg3:%arg4, %arg5:%arg6:%arg7) shape %10 : (!fir.box>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box> +! CHECK: %[[DES_V2:.*]] = hlfir.designate %[[DECL_V2]]#0 (%arg2:%arg3:%arg4, %arg5:%arg6:%arg7) shape %10 : (!fir.box>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box> +! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<2>) -> !hlfir.expr { +! CHECK: ^bb0(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index): +! CHECK: %[[D1:.*]] = hlfir.designate %13 (%[[ARG0]], %[[ARG1]]) : (!fir.box>, index, index) -> !fir.ref +! CHECK: %[[D2:.*]] = hlfir.designate %14 (%[[ARG0]], %[[ARG1]]) : (!fir.box>, index, index) -> !fir.ref +! CHECK: %[[LOAD1:.*]] = fir.load %[[D1]] : !fir.ref +! CHECK: %[[LOAD2:.*]] = fir.load %[[D2]] : !fir.ref +! CHECK: %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD1]], %[[LOAD2]] {{.*}} : f32 +! CHECK: %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : f32 +! CHECK: hlfir.yield_element %[[SELECT]] : f32 +! CHECK: } +! CHECK: hlfir.assign %[[ELEMENTAL]] to %[[DECL_V1]]#0 : !hlfir.expr, !fir.box> +! CHECK: acc.yield %[[V1]] : !fir.ref> ! CHECK: } ! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_ptr_Uxf32 : !fir.box>> reduction_operator init { @@ -40,111 +39,111 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_heap_Uxf32 : !fir.box>> reduction_operator init { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box>>): -! HLFIR: %[[CST:.*]] = arith.constant -1.401300e-45 : f32 -! HLFIR: %[[C0:.*]] = arith.constant 0 : index -! HLFIR: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box>>, index) -> (index, index, index) -! HLFIR: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> -! HLFIR: %[[TEMP:.*]] = fir.allocmem !fir.array, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""} -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %2(%1) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) -! HLFIR: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.box> +! CHECK: %[[CST:.*]] = arith.constant -1.401300e-45 : f32 +! CHECK: %[[C0:.*]] = arith.constant 0 : index +! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box>>, index) -> (index, index, index) +! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> +! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""} +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %2(%1) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) +! CHECK: hlfir.assign %[[CST]] to %[[DECLARE]]#0 : f32, !fir.box> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box> ! CHECK: } combiner { -! HLFIR: ^bb0(%[[ARG0:.*]]: !fir.box>>, %[[ARG1:.*]]: !fir.box>>, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index): -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> -! HLFIR: %[[DES_V1:.*]] = hlfir.designate %[[ARG0]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box>>, index, index, index, !fir.shape<1>) -> !fir.box>> -! HLFIR: %[[DES_V2:.*]] = hlfir.designate %[[ARG1]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box>>, index, index, index, !fir.shape<1>) -> !fir.box>> -! HLFIR: %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr { -! HLFIR: ^bb0(%[[IV:.*]]: index): -! HLFIR: %[[V1:.*]] = hlfir.designate %[[DES_V1]] (%[[IV]]) : (!fir.box>>, index) -> !fir.ref -! HLFIR: %[[V2:.*]] = hlfir.designate %[[DES_V2]] (%[[IV]]) : (!fir.box>>, index) -> !fir.ref -! HLFIR: %[[LOAD_V1:.*]] = fir.load %[[V1]] : !fir.ref -! HLFIR: %[[LOAD_V2:.*]] = fir.load %[[V2]] : !fir.ref -! HLFIR: %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD_V1]], %[[LOAD_V2]] {{.*}} : f32 -! HLFIR: %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD_V1]], %[[LOAD_V2]] : f32 -! HLFIR: hlfir.yield_element %[[SELECT]] : f32 -! HLFIR: } -! HLFIR: hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr, !fir.box>> -! HLFIR: acc.yield %[[ARG0]] : !fir.box>> +! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box>>, %[[ARG1:.*]]: !fir.box>>, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index): +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> +! CHECK: %[[DES_V1:.*]] = hlfir.designate %[[ARG0]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box>>, index, index, index, !fir.shape<1>) -> !fir.box>> +! CHECK: %[[DES_V2:.*]] = hlfir.designate %[[ARG1]] (%[[ARG2]]:%[[ARG3]]:%[[ARG4]]) shape %[[SHAPE]] : (!fir.box>>, index, index, index, !fir.shape<1>) -> !fir.box>> +! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr { +! CHECK: ^bb0(%[[IV:.*]]: index): +! CHECK: %[[V1:.*]] = hlfir.designate %[[DES_V1]] (%[[IV]]) : (!fir.box>>, index) -> !fir.ref +! CHECK: %[[V2:.*]] = hlfir.designate %[[DES_V2]] (%[[IV]]) : (!fir.box>>, index) -> !fir.ref +! CHECK: %[[LOAD_V1:.*]] = fir.load %[[V1]] : !fir.ref +! CHECK: %[[LOAD_V2:.*]] = fir.load %[[V2]] : !fir.ref +! CHECK: %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD_V1]], %[[LOAD_V2]] {{.*}} : f32 +! CHECK: %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD_V1]], %[[LOAD_V2]] : f32 +! CHECK: hlfir.yield_element %[[SELECT]] : f32 +! CHECK: } +! CHECK: hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr, !fir.box>> +! CHECK: acc.yield %[[ARG0]] : !fir.box>> ! CHECK: } ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_lb1.ub3_box_Uxi32 : !fir.box> reduction_operator init { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box>): -! HLFIR: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %c0{{.*}} : (!fir.box>, index) -> (index, index, index) -! HLFIR: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> -! HLFIR: %[[TEMP:.*]] = fir.allocmem !fir.array, %0#1 {bindc_name = ".tmp", uniq_name = ""} -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) -! HLFIR: hlfir.assign %c0{{.*}} to %[[DECLARE]]#0 : i32, !fir.box> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.box> +! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %c0{{.*}} : (!fir.box>, index) -> (index, index, index) +! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> +! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array, %0#1 {bindc_name = ".tmp", uniq_name = ""} +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) +! CHECK: hlfir.assign %c0{{.*}} to %[[DECLARE]]#0 : i32, !fir.box> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box>, %[[ARG1:.*]]: !fir.box>): -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> -! HLFIR: %[[DES1:.*]] = hlfir.designate %[[ARG0]] shape %[[SHAPE]] : (!fir.box>, !fir.shape<1>) -> !fir.box> -! HLFIR: %[[DES2:.*]] = hlfir.designate %[[ARG1]] shape %[[SHAPE]] : (!fir.box>, !fir.shape<1>) -> !fir.box> -! HLFIR: %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr { -! HLFIR: ^bb0(%[[IV:.*]]: index): -! HLFIR: %[[DES_V1:.*]] = hlfir.designate %[[DES1]] (%[[IV]]) : (!fir.box>, index) -> !fir.ref -! HLFIR: %[[DES_V2:.*]] = hlfir.designate %[[DES2]] (%[[IV]]) : (!fir.box>, index) -> !fir.ref -! HLFIR: %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref -! HLFIR: %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref -! HLFIR: %[[COMBINED:.*]] = arith.addi %[[LOAD_V1]], %[[LOAD_V2]] : i32 -! HLFIR: hlfir.yield_element %[[COMBINED]] : i32 -! HLFIR: } -! HLFIR: hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr, !fir.box> -! HLFIR: acc.yield %[[ARG0]] : !fir.box> -! HLFIR: } +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> +! CHECK: %[[DES1:.*]] = hlfir.designate %[[ARG0]] shape %[[SHAPE]] : (!fir.box>, !fir.shape<1>) -> !fir.box> +! CHECK: %[[DES2:.*]] = hlfir.designate %[[ARG1]] shape %[[SHAPE]] : (!fir.box>, !fir.shape<1>) -> !fir.box> +! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<1>) -> !hlfir.expr { +! CHECK: ^bb0(%[[IV:.*]]: index): +! CHECK: %[[DES_V1:.*]] = hlfir.designate %[[DES1]] (%[[IV]]) : (!fir.box>, index) -> !fir.ref +! CHECK: %[[DES_V2:.*]] = hlfir.designate %[[DES2]] (%[[IV]]) : (!fir.box>, index) -> !fir.ref +! CHECK: %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref +! CHECK: %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref +! CHECK: %[[COMBINED:.*]] = arith.addi %[[LOAD_V1]], %[[LOAD_V2]] : i32 +! CHECK: hlfir.yield_element %[[COMBINED]] : i32 +! CHECK: } +! CHECK: hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr, !fir.box> +! CHECK: acc.yield %[[ARG0]] : !fir.box> +! CHECK: } ! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_Uxf32 : !fir.box> reduction_operator init { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box>): ! CHECK: %[[INIT_VALUE:.*]] = arith.constant -1.401300e-45 : f32 -! HLFIR: %[[C0:.*]] = arith.constant 0 : index -! HLFIR: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box>, index) -> (index, index, index) -! HLFIR: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> -! HLFIR: %[[TEMP:.*]] = fir.allocmem !fir.array, %0#1 {bindc_name = ".tmp", uniq_name = ""} -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) -! HLFIR: hlfir.assign %[[INIT_VALUE]] to %[[DECLARE]]#0 : f32, !fir.box> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.box> +! CHECK: %[[C0:.*]] = arith.constant 0 : index +! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box>, index) -> (index, index, index) +! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> +! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array, %0#1 {bindc_name = ".tmp", uniq_name = ""} +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) +! CHECK: hlfir.assign %[[INIT_VALUE]] to %[[DECLARE]]#0 : f32, !fir.box> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box>, %[[ARG1:.*]]: !fir.box> -! HLFIR: %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> -! HLFIR: %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> -! HLFIR: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr { -! HLFIR: ^bb0(%{{.*}}: index): -! HLFIR: %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref -! HLFIR: %[[DES_V2:.*]] = hlfir.designate %[[RIGHT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref -! HLFIR: %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref -! HLFIR: %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref -! HLFIR: %[[CMPF:.*]] = arith.cmpf ogt, %[[LOAD_V1]], %[[LOAD_V2]] {{.*}} : f32 -! HLFIR: %[[SELECT:.*]] = arith.select %[[CMPF]], %[[LOAD_V1]], %[[LOAD_V2]] : f32 -! HLFIR: hlfir.yield_element %[[SELECT]] : f32 -! HLFIR: } -! HLFIR: hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr, !fir.box> +! CHECK: %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> +! CHECK: %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> +! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr { +! CHECK: ^bb0(%{{.*}}: index): +! CHECK: %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref +! CHECK: %[[DES_V2:.*]] = hlfir.designate %[[RIGHT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref +! CHECK: %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref +! CHECK: %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref +! CHECK: %[[CMPF:.*]] = arith.cmpf ogt, %[[LOAD_V1]], %[[LOAD_V2]] {{.*}} : f32 +! CHECK: %[[SELECT:.*]] = arith.select %[[CMPF]], %[[LOAD_V1]], %[[LOAD_V2]] : f32 +! CHECK: hlfir.yield_element %[[SELECT]] : f32 +! CHECK: } +! CHECK: hlfir.assign %[[ELEMENTAL]] to %[[ARG0]] : !hlfir.expr, !fir.box> ! CHECK: acc.yield %[[ARG0]] : !fir.box> ! CHECK: } ! CHECK-LABEL: acc.reduction.recipe @reduction_add_box_Uxi32 : !fir.box> reduction_operator init { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.box>): -! HLFIR: %[[INIT_VALUE:.*]] = arith.constant 0 : i32 -! HLFIR: %[[C0:.*]] = arith.constant 0 : index -! HLFIR: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box>, index) -> (index, index, index) -! HLFIR: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> -! HLFIR: %[[TEMP:.*]] = fir.allocmem !fir.array, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""} -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) -! HLFIR: hlfir.assign %[[INIT_VALUE]] to %[[DECLARE]]#0 : i32, !fir.box> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.box> +! CHECK: %[[INIT_VALUE:.*]] = arith.constant 0 : i32 +! CHECK: %[[C0:.*]] = arith.constant 0 : index +! CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box>, index) -> (index, index, index) +! CHECK: %[[SHAPE:.*]] = fir.shape %[[BOX_DIMS]]#1 : (index) -> !fir.shape<1> +! CHECK: %[[TEMP:.*]] = fir.allocmem !fir.array, %[[BOX_DIMS]]#1 {bindc_name = ".tmp", uniq_name = ""} +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = ".tmp"} : (!fir.heap>, !fir.shape<1>) -> (!fir.box>, !fir.heap>) +! CHECK: hlfir.assign %[[INIT_VALUE]] to %[[DECLARE]]#0 : i32, !fir.box> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.box> ! CHECK: } combiner { ! CHECK: ^bb0(%[[V1:.*]]: !fir.box>, %[[V2:.*]]: !fir.box> -! HLFIR: %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> -! HLFIR: %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> -! HLFIR: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr { -! HLFIR: ^bb0(%{{.*}}: index): -! HLFIR: %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref -! HLFIR: %[[DES_V2:.*]] = hlfir.designate %[[RIGHT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref -! HLFIR: %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref -! HLFIR: %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref -! HLFIR: %[[COMBINED:.*]] = arith.addi %[[LOAD_V1]], %[[LOAD_V2]] : i32 -! HLFIR: hlfir.yield_element %[[COMBINED]] : i32 -! HLFIR: } -! HLFIR: hlfir.assign %[[ELEMENTAL]] to %[[V1]] : !hlfir.expr, !fir.box> +! CHECK: %[[LEFT:.*]] = hlfir.designate %[[ARG0]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> +! CHECK: %[[RIGHT:.*]] = hlfir.designate %[[ARG1]] (%{{.*}}:%{{.*}}:%{{.*}}) shape %{{.*}} : (!fir.box>, index, index, index, !fir.shape<1>) -> !fir.box> +! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr { +! CHECK: ^bb0(%{{.*}}: index): +! CHECK: %[[DES_V1:.*]] = hlfir.designate %[[LEFT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref +! CHECK: %[[DES_V2:.*]] = hlfir.designate %[[RIGHT]] (%{{.*}}) : (!fir.box>, index) -> !fir.ref +! CHECK: %[[LOAD_V1:.*]] = fir.load %[[DES_V1]] : !fir.ref +! CHECK: %[[LOAD_V2:.*]] = fir.load %[[DES_V2]] : !fir.ref +! CHECK: %[[COMBINED:.*]] = arith.addi %[[LOAD_V1]], %[[LOAD_V2]] : i32 +! CHECK: hlfir.yield_element %[[COMBINED]] : i32 +! CHECK: } +! CHECK: hlfir.assign %[[ELEMENTAL]] to %[[V1]] : !hlfir.expr, !fir.box> ! CHECK: acc.yield %arg0 : !fir.box> ! CHECK: } @@ -156,9 +155,9 @@ ! CHECK: %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4> ! CHECK: %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.complex<4> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) -! HLFIR: fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref> @@ -176,9 +175,9 @@ ! CHECK: %[[UNDEF1:.*]] = fir.insert_value %[[UNDEF]], %[[REAL]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4> ! CHECK: %[[UNDEF2:.*]] = fir.insert_value %[[UNDEF1]], %[[IMAG]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.complex<4> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) -! HLFIR: fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: fir.store %[[UNDEF2]] to %[[DECLARE]]#0 : !fir.ref> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref> @@ -192,10 +191,10 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[CST:.*]] = arith.constant false ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4> -! HLFIR: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref> @@ -212,10 +211,10 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[CST:.*]] = arith.constant true ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4> -! HLFIR: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref> @@ -232,10 +231,10 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[CST:.*]] = arith.constant false ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4> -! HLFIR: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref> @@ -252,10 +251,10 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[CST:.*]] = arith.constant true ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.logical<4> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[CONVERT:.*]] = fir.convert %[[CST]] : (i1) -> !fir.logical<4> -! HLFIR: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: fir.store %[[CONVERT]] to %[[DECLARE]]#0 : !fir.ref> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref> @@ -272,9 +271,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[CST:.*]] = arith.constant 0 : i32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 -! HLFIR: %[[DECLARE]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[CST]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[CST]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -288,9 +287,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[CST:.*]] = arith.constant 0 : i32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[CST]] to %[[DECLARE:.*]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE:.*]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[CST]] to %[[DECLARE:.*]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE:.*]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -304,9 +303,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[CST:.*]] = arith.constant -1 : i32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[CST]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[CST]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -319,17 +318,17 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100_ref_100xf32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant -1.401300e-45 : f32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) ! CHECK: %[[LB:.*]] = arith.constant 0 : index ! CHECK: %[[UB:.*]] = arith.constant 99 : index ! CHECK: %[[STEP:.*]] = arith.constant 1 : index ! CHECK: fir.do_loop %[[IV:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] { -! HLFIR: %[[COORD:.*]] = fir.coordinate_of %[[DECLARE]]#0, %[[IV]] : (!fir.ref>, index) -> !fir.ref -! HLFIR: fir.store %[[INIT]] to %[[COORD]] : !fir.ref +! CHECK: %[[COORD:.*]] = fir.coordinate_of %[[DECLARE]]#0, %[[IV]] : (!fir.ref>, index) -> !fir.ref +! CHECK: fir.store %[[INIT]] to %[[COORD]] : !fir.ref ! CHECK: } -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB0:.*]] = arith.constant 0 : index @@ -351,9 +350,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant -1.401300e-45 : f32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca f32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %0 {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %0 {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -367,10 +366,10 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100xext10_ref_100x10xi32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%arg0: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant -2147483648 : i32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.ref>, !fir.ref>) -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.ref>, !fir.ref>) +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB0:.*]] = arith.constant 0 : index @@ -397,9 +396,9 @@ ! CHECK: ^bb0(%arg0: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant -2147483648 : i32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -413,10 +412,10 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100xext10_ref_100x10xf32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant 3.40282347E+38 : f32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xf32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.ref>, !fir.ref>) -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.ref>, !fir.ref>) +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB0:.*]] = arith.constant 0 : index @@ -443,9 +442,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant 3.40282347E+38 : f32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca f32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -459,10 +458,10 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100_ref_100xi32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant 2147483647 : i32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB0:.*]] = arith.constant 0 : index @@ -484,9 +483,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant 2147483647 : i32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -501,9 +500,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant 1.000000e+00 : f32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca f32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -516,10 +515,10 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_mul_section_ext100_ref_100xi32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant 1 : i32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB:.*]] = arith.constant 0 : index @@ -540,9 +539,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant 1 : i32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -555,10 +554,10 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xf32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant 0.000000e+00 : f32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB:.*]] = arith.constant 0 : index @@ -579,9 +578,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant 0.000000e+00 : f32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca f32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -594,10 +593,10 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant 0 : i32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10x2xi32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<3>) -> (!fir.ref>, !fir.ref>) -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<3>) -> (!fir.ref>, !fir.ref>) +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB0:.*]] = arith.constant 0 : index @@ -627,10 +626,10 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10_ref_100x10xi32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant 0 : i32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.ref>, !fir.ref>) -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref> +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<2>) -> (!fir.ref>, !fir.ref>) +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): ! CHECK: %[[LB0:.*]] = arith.constant 0 : index @@ -655,9 +654,9 @@ ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xi32 : !fir.ref> reduction_operator init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[INIT:.*]] = arith.constant 0 : i32 -! HLFIR: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> +! CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1> ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32> -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) ! HFLIR: acc.yield %[[DECLARE]]#0 : !fir.ref> ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref>, %[[ARG1:.*]]: !fir.ref>): @@ -679,9 +678,9 @@ ! CHECK: ^bb0(%{{.*}}: !fir.ref): ! CHECK: %[[INIT:.*]] = arith.constant 0 : i32 ! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 -! HLFIR: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) -! HLFIR: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref -! HLFIR: acc.yield %[[DECLARE]]#0 : !fir.ref +! CHECK: %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.reduction.init"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.store %[[INIT]] to %[[DECLARE]]#0 : !fir.ref +! CHECK: acc.yield %[[DECLARE]]#0 : !fir.ref ! CHECK: } combiner { ! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref, %[[ARG1:.*]]: !fir.ref): ! CHECK: %[[LOAD0:.*]] = fir.load %[[ARG0]] : !fir.ref @@ -703,9 +702,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_int( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_ref_i32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_add_int_array_1d(a, b) @@ -720,9 +718,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_1d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref>) subroutine acc_reduction_add_int_array_2d(a, b) @@ -739,9 +736,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_2d( ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref> {fir.bindc_name = "b"}) { -! HLFIR: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] -! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] +! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref>) { ! CHECK: } attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} @@ -761,9 +757,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_int_array_3d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] -! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] +! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 -> %[[RED_ARG1]] : !fir.ref>) ! CHECK: } attributes {collapse = [3], collapseDeviceType = [#acc.device_type]} @@ -779,9 +774,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_float( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_ref_f32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_add_float_array_1d(a, b) @@ -796,9 +790,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_float_array_1d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_add_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref>) subroutine acc_reduction_mul_int(a, b) @@ -813,9 +806,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_mul_int( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_mul_ref_i32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_mul_int_array_1d(a, b) @@ -830,9 +822,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_mul_int_array_1d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_mul_section_ext100_ref_100xi32 -> %[[RED_B]] : !fir.ref>) subroutine acc_reduction_mul_float(a, b) @@ -847,9 +838,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_mul_float( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_mul_ref_f32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_mul_float_array_1d(a, b) @@ -864,9 +854,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_mul_float_array_1d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref>) bounds(%2) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_mul_section_ext100_ref_100xf32 -> %[[RED_B]] : !fir.ref>) subroutine acc_reduction_min_int(a, b) @@ -881,9 +870,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_min_int( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_min_ref_i32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_min_int_array_1d(a, b) @@ -898,9 +886,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_min_int_array_1d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] -! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%2) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] +! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_min_section_ext100_ref_100xi32 -> %[[RED_ARG1]] : !fir.ref>) subroutine acc_reduction_min_float(a, b) @@ -915,9 +902,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_min_float( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_min_ref_f32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_min_float_array2d(a, b) @@ -934,9 +920,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_min_float_array2d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] -! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%3, %5) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] +! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_min_section_ext100xext10_ref_100x10xf32 -> %[[RED_ARG1]] : !fir.ref>) ! CHECK: attributes {collapse = [2], collapseDeviceType = [#acc.device_type]} @@ -952,9 +937,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_max_int( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_max_ref_i32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_max_int_array2d(a, b) @@ -971,9 +955,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_max_int_array2d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] -! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] +! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_max_section_ext100xext10_ref_100x10xi32 -> %[[RED_ARG1]] : !fir.ref>) subroutine acc_reduction_max_float(a, b) @@ -988,9 +971,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_max_float( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref {fir.bindc_name = "b"}) -! HLFIR: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] -! FIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[B]] : !fir.ref) -> !fir.ref {name = "b"} -! HLFIR: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} +! CHECK: %[[DECLB:.*]]:2 = hlfir.declare %[[B]] +! CHECK: %[[RED_B:.*]] = acc.reduction varPtr(%[[DECLB]]#1 : !fir.ref) -> !fir.ref {name = "b"} ! CHECK: acc.loop reduction(@reduction_max_ref_f32 -> %[[RED_B]] : !fir.ref) subroutine acc_reduction_max_float_array1d(a, b) @@ -1005,9 +987,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_max_float_array1d( ! CHECK-SAME: %{{.*}}: !fir.ref> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref> {fir.bindc_name = "b"}) -! HLFIR: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] -! FIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[ARG1]] : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} -! HLFIR: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} +! CHECK: %[[DECLARG1:.*]]:2 = hlfir.declare %[[ARG1]] +! CHECK: %[[RED_ARG1:.*]] = acc.reduction varPtr(%[[DECLARG1]]#1 : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "b"} ! CHECK: acc.loop reduction(@reduction_max_section_ext100_ref_100xf32 -> %[[RED_ARG1]] : !fir.ref>) { subroutine acc_reduction_iand() @@ -1048,9 +1029,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_and() ! CHECK: %[[L:.*]] = fir.alloca !fir.logical<4> {bindc_name = "l", uniq_name = "_QFacc_reduction_andEl"} -! HLFIR: %[[DECLL:.*]]:2 = hlfir.declare %[[L]] -! FIR: %[[RED:.*]] = acc.reduction varPtr(%[[L]] : !fir.ref>) -> !fir.ref> {name = "l"} -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[DECLL]]#1 : !fir.ref>) -> !fir.ref> {name = "l"} +! CHECK: %[[DECLL:.*]]:2 = hlfir.declare %[[L]] +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLL]]#1 : !fir.ref>) -> !fir.ref> {name = "l"} ! CHECK: acc.parallel reduction(@reduction_land_ref_l32 -> %[[RED]] : !fir.ref>) subroutine acc_reduction_or() @@ -1112,9 +1092,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_alloc() ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.box> {bindc_name = "i", uniq_name = "_QFacc_reduction_add_allocEi"} -! HLFIR: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]] -! FIR: %[[LOAD:.*]] = fir.load %[[ALLOCA]] : !fir.ref>> -! HLFIR: %[[LOAD:.*]] = fir.load %[[DECL]]#1 : !fir.ref>> +! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOCA]] +! CHECK: %[[LOAD:.*]] = fir.load %[[DECL]]#1 : !fir.ref>> ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box>) -> !fir.heap ! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap) -> !fir.heap {name = "i"} ! CHECK: acc.parallel reduction(@reduction_add_heap_i32 -> %[[RED]] : !fir.heap) @@ -1127,9 +1106,8 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer( ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref>> {fir.bindc_name = "i"}) -! HLFIR: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] -! FIR: %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref>> -! HLFIR: %[[LOAD:.*]] = fir.load %[[DECLARG0]]#1 : !fir.ref>> +! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] +! CHECK: %[[LOAD:.*]] = fir.load %[[DECLARG0]]#1 : !fir.ref>> ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box>) -> !fir.ptr ! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr) -> !fir.ptr {name = "i"} ! CHECK: acc.parallel reduction(@reduction_add_ptr_i32 -> %[[RED]] : !fir.ptr) @@ -1143,13 +1121,12 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_static_slice( ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref> {fir.bindc_name = "a"}) ! CHECK: %[[C100:.*]] = arith.constant 100 : index -! HLFIR: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] +! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] ! CHECK: %[[C1:.*]] = arith.constant 1 : index ! CHECK: %[[LB:.*]] = arith.constant 10 : index ! CHECK: %[[UB:.*]] = arith.constant 19 : index ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[C100]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index) -! FIR: %[[RED:.*]] = acc.reduction varPtr(%[[ARG0]] : !fir.ref>) bounds(%[[BOUND]]) -> !fir.ref> {name = "a(11:20)"} -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#1 : !fir.ref>) bounds(%[[BOUND]]) -> !fir.ref> {name = "a(11:20)"} +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#1 : !fir.ref>) bounds(%[[BOUND]]) -> !fir.ref> {name = "a(11:20)"} ! CHECK: acc.parallel reduction(@reduction_add_section_lb10.ub19_ref_100xi32 -> %[[RED]] : !fir.ref>) subroutine acc_reduction_add_dynamic_extent_add(a) @@ -1160,9 +1137,9 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add( ! CHECK-SAME: %[[ARG0:.*]]: !fir.box> {fir.bindc_name = "a"}) -! HLFIR: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "a"} -! HLFIR: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.ref>) +! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "a"} +! CHECK: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.ref>) subroutine acc_reduction_add_assumed_shape_max(a) real :: a(:) @@ -1172,9 +1149,9 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_assumed_shape_max( ! CHECK-SAME: %[[ARG0:.*]]: !fir.box> {fir.bindc_name = "a"}) -! HLFIR: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "a"} -! HLFIR: acc.parallel reduction(@reduction_max_box_Uxf32 -> %[[RED]] : !fir.ref>) { +! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]] +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref>) bounds(%{{.*}}) -> !fir.ref> {name = "a"} +! CHECK: acc.parallel reduction(@reduction_max_box_Uxf32 -> %[[RED]] : !fir.ref>) { subroutine acc_reduction_add_dynamic_extent_add_with_section(a) integer :: a(:) @@ -1184,11 +1161,11 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_add_with_section( ! CHECK-SAME: %[[ARG0:.*]]: !fir.box> {fir.bindc_name = "a"}) -! HLFIR: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box>) -> (!fir.box>, !fir.box>) -! HLFIR: %[[BOUND:.*]] = acc.bounds lowerbound(%c1{{.*}} : index) upperbound(%c3{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true} -! HLFIR: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL]]#1 : (!fir.box>) -> !fir.ref> -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ref>) bounds(%[[BOUND]]) -> !fir.ref> {name = "a(2:4)"} -! HLFIR: acc.parallel reduction(@reduction_add_section_lb1.ub3_box_Uxi32 -> %[[RED]] : !fir.ref>) +! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFacc_reduction_add_dynamic_extent_add_with_sectionEa"} : (!fir.box>) -> (!fir.box>, !fir.box>) +! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c1{{.*}} : index) upperbound(%c3{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}} : index) {strideInBytes = true} +! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL]]#1 : (!fir.box>) -> !fir.ref> +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ref>) bounds(%[[BOUND]]) -> !fir.ref> {name = "a(2:4)"} +! CHECK: acc.parallel reduction(@reduction_add_section_lb1.ub3_box_Uxi32 -> %[[RED]] : !fir.ref>) subroutine acc_reduction_add_allocatable(a) real, allocatable :: a(:) @@ -1198,12 +1175,12 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_allocatable( ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref>>> {fir.bindc_name = "a"}) -! HLFIR: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFacc_reduction_add_allocatableEa"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) -! HLFIR: %[[BOX:.*]] = fir.load %[[DECL]]#1 : !fir.ref>>> -! HLFIR: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true} -! HLFIR: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box>>) -> !fir.heap> -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap>) bounds(%6) -> !fir.heap> {name = "a"} -! HLFIR: acc.parallel reduction(@reduction_max_box_heap_Uxf32 -> %[[RED]] : !fir.heap>) +! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFacc_reduction_add_allocatableEa"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#1 : !fir.ref>>> +! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true} +! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box>>) -> !fir.heap> +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.heap>) bounds(%6) -> !fir.heap> {name = "a"} +! CHECK: acc.parallel reduction(@reduction_max_box_heap_Uxf32 -> %[[RED]] : !fir.heap>) subroutine acc_reduction_add_pointer_array(a) real, pointer :: a(:) @@ -1213,12 +1190,12 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_add_pointer_array( ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref>>> {fir.bindc_name = "a"}) -! HLFIR: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFacc_reduction_add_pointer_arrayEa"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) -! HLFIR: %[[BOX:.*]] = fir.load %[[DECL]]#1 : !fir.ref>>> -! HLFIR: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true} -! HLFIR: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box>>) -> !fir.ptr> -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr>) bounds(%[[BOUND]]) -> !fir.ptr> {name = "a"} -! HLFIR: acc.parallel reduction(@reduction_max_box_ptr_Uxf32 -> %[[RED]] : !fir.ptr>) +! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFacc_reduction_add_pointer_arrayEa"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[BOX:.*]] = fir.load %[[DECL]]#1 : !fir.ref>>> +! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}}#1 : index) stride(%{{.*}}#2 : index) startIdx(%{{.*}}#0 : index) {strideInBytes = true} +! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box>>) -> !fir.ptr> +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr>) bounds(%[[BOUND]]) -> !fir.ptr> {name = "a"} +! CHECK: acc.parallel reduction(@reduction_max_box_ptr_Uxf32 -> %[[RED]] : !fir.ptr>) subroutine acc_reduction_max_dynamic_extent_max(a, n) integer :: n @@ -1229,6 +1206,6 @@ end subroutine ! CHECK-LABEL: func.func @_QPacc_reduction_max_dynamic_extent_max( ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref> {fir.bindc_name = "a"}, %{{.*}}: !fir.ref {fir.bindc_name = "n"}) -! HLFIR: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) -! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[DECL_A]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "a"} -! HLFIR: acc.parallel reduction(@reduction_max_ref_UxUxf32 -> %[[RED]] : !fir.ref>) +! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref>, !fir.shape<2>) -> (!fir.box>, !fir.ref>) +! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECL_A]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {name = "a"} +! CHECK: acc.parallel reduction(@reduction_max_ref_UxUxf32 -> %[[RED]] : !fir.ref>) -- GitLab From 23aabdd66f78dc28919c4a85d4bdcbf7b0dd89f7 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 5 Jan 2024 18:40:40 -0800 Subject: [PATCH 178/728] [NFC][sanitizer] Move SymbolizedStackHolder into sanitizer_common (#77152) And replace most `ClearAll()` uses. --- compiler-rt/lib/asan/asan_suppressions.cpp | 7 +++---- compiler-rt/lib/hwasan/hwasan_report.cpp | 6 ++++-- compiler-rt/lib/lsan/lsan_common.cpp | 7 ++++--- .../sanitizer_stacktrace_libcdep.cpp | 12 +++++------ .../sanitizer_common/sanitizer_symbolizer.h | 20 +++++++++++++++++++ .../sanitizer_symbolizer_report.cpp | 5 +++-- compiler-rt/lib/ubsan/ubsan_diag.h | 20 ------------------- 7 files changed, 40 insertions(+), 37 deletions(-) diff --git a/compiler-rt/lib/asan/asan_suppressions.cpp b/compiler-rt/lib/asan/asan_suppressions.cpp index 8cb2c3e3b9b6..e71d23182186 100644 --- a/compiler-rt/lib/asan/asan_suppressions.cpp +++ b/compiler-rt/lib/asan/asan_suppressions.cpp @@ -81,9 +81,10 @@ bool IsStackTraceSuppressed(const StackTrace *stack) { } if (suppression_ctx->HasSuppressionType(kInterceptorViaFunction)) { - SymbolizedStack *frames = symbolizer->SymbolizePC(addr); + SymbolizedStackHolder symbolized_stack(symbolizer->SymbolizePC(addr)); + const SymbolizedStack *frames = symbolized_stack.get(); CHECK(frames); - for (SymbolizedStack *cur = frames; cur; cur = cur->next) { + for (const SymbolizedStack *cur = frames; cur; cur = cur->next) { const char *function_name = cur->info.function; if (!function_name) { continue; @@ -91,11 +92,9 @@ bool IsStackTraceSuppressed(const StackTrace *stack) { // Match "interceptor_via_fun" suppressions. if (suppression_ctx->Match(function_name, kInterceptorViaFunction, &s)) { - frames->ClearAll(); return true; } } - frames->ClearAll(); } } return false; diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp index 1a018a891b56..784cfb904aa2 100644 --- a/compiler-rt/lib/hwasan/hwasan_report.cpp +++ b/compiler-rt/lib/hwasan/hwasan_report.cpp @@ -292,12 +292,14 @@ static void PrintStackAllocations(const StackAllocationsRingBuffer *sa, uptr pc = record & pc_mask; frame_desc.AppendF(" record_addr:0x%zx record:0x%zx", reinterpret_cast(record_addr), record); - if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { + SymbolizedStackHolder symbolized_stack( + Symbolizer::GetOrInit()->SymbolizePC(pc)); + const SymbolizedStack *frame = symbolized_stack.get(); + if (frame) { StackTracePrinter::GetOrInit()->RenderFrame( &frame_desc, " %F %L", 0, frame->info.address, &frame->info, common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix); - frame->ClearAll(); } Printf("%s\n", frame_desc.data()); frame_desc.clear(); diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp index e24839c984b3..0ecded8b28cd 100644 --- a/compiler-rt/lib/lsan/lsan_common.cpp +++ b/compiler-rt/lib/lsan/lsan_common.cpp @@ -155,14 +155,15 @@ Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) { return s; // Suppress by file or function name. - SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr); - for (SymbolizedStack *cur = frames; cur; cur = cur->next) { + SymbolizedStackHolder symbolized_stack( + Symbolizer::GetOrInit()->SymbolizePC(addr)); + const SymbolizedStack *frames = symbolized_stack.get(); + for (const SymbolizedStack *cur = frames; cur; cur = cur->next) { if (context.Match(cur->info.function, kSuppressionLeak, &s) || context.Match(cur->info.file, kSuppressionLeak, &s)) { break; } } - frames->ClearAll(); return s; } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp index 9a4c80fcfdd1..561eae9ab780 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cpp @@ -33,13 +33,14 @@ class StackTraceTextPrinter { stack_trace_fmt)) {} bool ProcessAddressFrames(uptr pc) { - SymbolizedStack *frames = symbolize_ - ? Symbolizer::GetOrInit()->SymbolizePC(pc) - : SymbolizedStack::New(pc); + SymbolizedStackHolder symbolized_stack( + symbolize_ ? Symbolizer::GetOrInit()->SymbolizePC(pc) + : SymbolizedStack::New(pc)); + const SymbolizedStack *frames = symbolized_stack.get(); if (!frames) return false; - for (SymbolizedStack *cur = frames; cur; cur = cur->next) { + for (const SymbolizedStack *cur = frames; cur; cur = cur->next) { uptr prev_len = output_->length(); StackTracePrinter::GetOrInit()->RenderFrame( output_, stack_trace_fmt_, frame_num_++, cur->info.address, @@ -51,13 +52,12 @@ class StackTraceTextPrinter { ExtendDedupToken(cur); } - frames->ClearAll(); return true; } private: // Extend the dedup token by appending a new frame. - void ExtendDedupToken(SymbolizedStack *stack) { + void ExtendDedupToken(const SymbolizedStack *stack) { if (!dedup_token_) return; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h index 82cd9bc22791..16ef2f2fd717 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h @@ -64,6 +64,26 @@ struct SymbolizedStack { SymbolizedStack(); }; +class SymbolizedStackHolder { + SymbolizedStack *Stack; + + void clear() { + if (Stack) + Stack->ClearAll(); + } + + public: + explicit SymbolizedStackHolder(SymbolizedStack *Stack = nullptr) + : Stack(Stack) {} + ~SymbolizedStackHolder() { clear(); } + void reset(SymbolizedStack *S = nullptr) { + if (Stack != S) + clear(); + Stack = S; + } + const SymbolizedStack *get() const { return Stack; } +}; + // For now, DataInfo is used to describe global variable. struct DataInfo { // Owns all the string members. Storage for them is diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp index ec60dd3e0d66..26f015e70d04 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -102,9 +102,10 @@ void ReportErrorSummary(const char *error_type, const StackTrace *stack, // Currently, we include the first stack frame into the report summary. // Maybe sometimes we need to choose another frame (e.g. skip memcpy/etc). uptr pc = StackTrace::GetPreviousInstructionPc(stack->trace[0]); - SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc); + SymbolizedStackHolder symbolized_stack( + Symbolizer::GetOrInit()->SymbolizePC(pc)); + const SymbolizedStack *frame = symbolized_stack.get(); ReportErrorSummary(error_type, frame->info, alt_tool_name); - frame->ClearAll(); #endif } diff --git a/compiler-rt/lib/ubsan/ubsan_diag.h b/compiler-rt/lib/ubsan/ubsan_diag.h index b444e971b228..c836647c98f3 100644 --- a/compiler-rt/lib/ubsan/ubsan_diag.h +++ b/compiler-rt/lib/ubsan/ubsan_diag.h @@ -18,26 +18,6 @@ namespace __ubsan { -class SymbolizedStackHolder { - SymbolizedStack *Stack; - - void clear() { - if (Stack) - Stack->ClearAll(); - } - -public: - explicit SymbolizedStackHolder(SymbolizedStack *Stack = nullptr) - : Stack(Stack) {} - ~SymbolizedStackHolder() { clear(); } - void reset(SymbolizedStack *S) { - if (Stack != S) - clear(); - Stack = S; - } - const SymbolizedStack *get() const { return Stack; } -}; - SymbolizedStack *getSymbolizedLocation(uptr PC); inline SymbolizedStack *getCallerLocation(uptr CallerPC) { -- GitLab From c7e4065aad78f77d61be1d1ac674546cc62208d1 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Fri, 5 Jan 2024 18:42:06 -0800 Subject: [PATCH 179/728] [NFC][sanitizer] Add consts to SkipInternalFrames (#77162) --- compiler-rt/lib/sanitizer_common/sanitizer_common.h | 2 +- .../lib/sanitizer_common/sanitizer_symbolizer_report.cpp | 4 ++-- compiler-rt/lib/tsan/rtl/tsan_report.cpp | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 7d9d61de8b61..b99c0cffcbb1 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -395,7 +395,7 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info, void ReportErrorSummary(const char *error_type, const StackTrace *trace, const char *alt_tool_name = nullptr); // Skips frames which we consider internal and not usefull to the users. -SymbolizedStack *SkipInternalFrames(SymbolizedStack *frames); +const SymbolizedStack *SkipInternalFrames(const SymbolizedStack *frames); void ReportMmapWriteExec(int prot, int mflags); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp index 26f015e70d04..0cf250f72129 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -41,8 +41,8 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { return false; } -SymbolizedStack *SkipInternalFrames(SymbolizedStack *frames) { - for (SymbolizedStack *f = frames; f; f = f->next) +const SymbolizedStack *SkipInternalFrames(const SymbolizedStack *frames) { + for (const SymbolizedStack *f = frames; f; f = f->next) if (!FrameIsInternal(f)) return f; return nullptr; diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp index b1cee7ac8cbf..22ba428cc58b 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp @@ -273,8 +273,8 @@ static ReportStack *ChooseSummaryStack(const ReportDesc *rep) { return 0; } -static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) { - if (SymbolizedStack *f = SkipInternalFrames(frames)) +static const SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) { + if (const SymbolizedStack *f = SkipInternalFrames(frames)) return f; return frames; // Fallback to the top frame. } @@ -350,7 +350,7 @@ void PrintReport(const ReportDesc *rep) { Printf(" And %d more similar thread leaks.\n\n", rep->count - 1); if (ReportStack *stack = ChooseSummaryStack(rep)) { - if (SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames)) + if (const SymbolizedStack *frame = SkipTsanInternalFrames(stack->frames)) ReportErrorSummary(rep_typ_str, frame->info); } -- GitLab From dbea538c4391caa8a369c0ccf720367f042185b1 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Fri, 5 Jan 2024 18:51:02 -0800 Subject: [PATCH 180/728] [DWARFLinker] Support MD5 checksums in the line table (#77151) Add support to the DWARF linkers for emitting DWARF 5 MD5 checksum in the line table. --- llvm/lib/DWARFLinker/DWARFStreamer.cpp | 24 +++++++++++++------ .../DebugLineSectionEmitter.h | 13 +++++++++- .../DWARFLinkerParallel/OutputSections.cpp | 4 ++++ llvm/lib/DWARFLinkerParallel/OutputSections.h | 2 ++ .../ARM/dwarf5-dwarf4-combination-macho.test | 12 +++++----- .../test/tools/dsymutil/ARM/dwarf5-macho.test | 9 +++---- 6 files changed, 46 insertions(+), 18 deletions(-) diff --git a/llvm/lib/DWARFLinker/DWARFStreamer.cpp b/llvm/lib/DWARFLinker/DWARFStreamer.cpp index faa91364ec9c..3ec082f4ea0c 100644 --- a/llvm/lib/DWARFLinker/DWARFStreamer.cpp +++ b/llvm/lib/DWARFLinker/DWARFStreamer.cpp @@ -859,10 +859,8 @@ void DwarfStreamer::emitLineTablePrologueV5IncludeAndFileTable( for (auto Include : P.IncludeDirectories) emitLineTableString(P, Include, DebugStrPool, DebugLineStrPool); - bool InlineSources = any_of(P.FileNames, [](auto &File) { - auto s = dwarf::toString(File.Source); - return s && !**s; - }); + bool HasChecksums = P.ContentTypes.HasMD5; + bool HasInlineSources = P.ContentTypes.HasSource; if (P.FileNames.empty()) { // file_name_entry_format_count (ubyte). @@ -870,7 +868,7 @@ void DwarfStreamer::emitLineTablePrologueV5IncludeAndFileTable( LineSectionSize += 1; } else { // file_name_entry_format_count (ubyte). - MS->emitInt8(2 + (InlineSources ? 1 : 0)); + MS->emitInt8(2 + (HasChecksums ? 1 : 0) + (HasInlineSources ? 1 : 0)); LineSectionSize += 1; // file_name_entry_format (sequence of ULEB128 pairs). @@ -880,7 +878,13 @@ void DwarfStreamer::emitLineTablePrologueV5IncludeAndFileTable( LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_LNCT_directory_index); LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_FORM_data1); - if (InlineSources) { + + if (HasChecksums) { + LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_LNCT_MD5); + LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_FORM_data16); + } + + if (HasInlineSources) { LineSectionSize += MS->emitULEB128IntValue(dwarf::DW_LNCT_LLVM_source); LineSectionSize += MS->emitULEB128IntValue(StrForm); } @@ -894,7 +898,13 @@ void DwarfStreamer::emitLineTablePrologueV5IncludeAndFileTable( emitLineTableString(P, File.Name, DebugStrPool, DebugLineStrPool); MS->emitInt8(File.DirIdx); LineSectionSize += 1; - if (InlineSources) + if (HasChecksums) { + MS->emitBinaryData( + StringRef(reinterpret_cast(File.Checksum.data()), + File.Checksum.size())); + LineSectionSize += File.Checksum.size(); + } + if (HasInlineSources) emitLineTableString(P, File.Source, DebugStrPool, DebugLineStrPool); } } diff --git a/llvm/lib/DWARFLinkerParallel/DebugLineSectionEmitter.h b/llvm/lib/DWARFLinkerParallel/DebugLineSectionEmitter.h index fc7f8cbc4a8e..27c63fad712f 100644 --- a/llvm/lib/DWARFLinkerParallel/DebugLineSectionEmitter.h +++ b/llvm/lib/DWARFLinkerParallel/DebugLineSectionEmitter.h @@ -197,7 +197,7 @@ private: Section.emitIntVal(0, 1); } else { // file_name_entry_format_count (ubyte). - Section.emitIntVal(2, 1); + Section.emitIntVal(2 + (P.ContentTypes.HasMD5 ? 1 : 0), 1); // file_name_entry_format (sequence of ULEB128 pairs). encodeULEB128(dwarf::DW_LNCT_path, Section.OS); @@ -205,6 +205,11 @@ private: encodeULEB128(dwarf::DW_LNCT_directory_index, Section.OS); encodeULEB128(dwarf::DW_FORM_data1, Section.OS); + + if (P.ContentTypes.HasMD5) { + encodeULEB128(dwarf::DW_LNCT_MD5, Section.OS); + encodeULEB128(dwarf::DW_FORM_data16, Section.OS); + } } // file_names_count (ULEB128). @@ -222,6 +227,12 @@ private: // source file. Section.emitString(File.Name.getForm(), *FileNameStr); Section.emitIntVal(File.DirIdx, 1); + + if (P.ContentTypes.HasMD5) { + Section.emitBinaryData( + StringRef(reinterpret_cast(File.Checksum.data()), + File.Checksum.size())); + } } } diff --git a/llvm/lib/DWARFLinkerParallel/OutputSections.cpp b/llvm/lib/DWARFLinkerParallel/OutputSections.cpp index 9c3e3ebd220a..730ae0f83d7b 100644 --- a/llvm/lib/DWARFLinkerParallel/OutputSections.cpp +++ b/llvm/lib/DWARFLinkerParallel/OutputSections.cpp @@ -227,6 +227,10 @@ void SectionDescriptor::emitIntVal(uint64_t Val, unsigned Size) { } } +void SectionDescriptor::emitBinaryData(llvm::StringRef Data) { + OS.write(Data.data(), Data.size()); +} + void SectionDescriptor::apply(uint64_t PatchOffset, dwarf::Form AttrForm, uint64_t Val) { switch (AttrForm) { diff --git a/llvm/lib/DWARFLinkerParallel/OutputSections.h b/llvm/lib/DWARFLinkerParallel/OutputSections.h index f23b2efb869d..0f394b0810ea 100644 --- a/llvm/lib/DWARFLinkerParallel/OutputSections.h +++ b/llvm/lib/DWARFLinkerParallel/OutputSections.h @@ -283,6 +283,8 @@ struct SectionDescriptor { void emitString(dwarf::Form StringForm, const char *StringVal); + void emitBinaryData(llvm::StringRef Data); + /// Emit specified inplace string value into the current section contents. void emitInplaceString(StringRef String) { OS << GlobalData.translateString(String); diff --git a/llvm/test/tools/dsymutil/ARM/dwarf5-dwarf4-combination-macho.test b/llvm/test/tools/dsymutil/ARM/dwarf5-dwarf4-combination-macho.test index 0199bf28681b..d5b78bd1487e 100644 --- a/llvm/test/tools/dsymutil/ARM/dwarf5-dwarf4-combination-macho.test +++ b/llvm/test/tools/dsymutil/ARM/dwarf5-dwarf4-combination-macho.test @@ -73,7 +73,7 @@ CHECK-NEXT: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000000) address = 0x[[ CHECK: DW_AT_linkage_name [DW_FORM_strx] (indexed (00000005) string = "_Z4foo2i") CHECK: DW_AT_name [DW_FORM_strx] (indexed (00000006) string = "foo2") CHECK: DW_TAG_formal_parameter -CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] (0x[[LOCLIST_OFFSET:[0-9a-f]+]]: +CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] (0x[[LOCLIST_OFFSET:[0-9a-f]+]]: CHECK-NEXT: [0x[[#%.16x,LOCLIST_PAIR_START:]], 0x[[#%.16x,LOCLIST_PAIR_END:]]): [[LOCLIST_EXPR:.*]] CHECK-NEXT: [0x[[#%.16x,LOCLIST_PAIR_START2:]], 0x[[#%.16x,LOCLIST_PAIR_END2:]]): [[LOCLIST_EXPR2:.*]]) CHECK: DW_AT_name [DW_FORM_strx] (indexed (00000007) string = "a") @@ -93,7 +93,7 @@ CHECK-NEXT: DW_AT_low_pc [DW_FORM_addr] (0x[[#%.16x,LOC_LOWPC CHECK: DW_AT_linkage_name [DW_FORM_strp] ( .debug_str[0x000000e6] = "_Z3bari") CHECK: DW_AT_name [DW_FORM_strp] ( .debug_str[0x000000ee] = "bar") CHECK: DW_TAG_formal_parameter -CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] (0x[[LOC_OFFSET:[0-9a-f]+]]: +CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] (0x[[LOC_OFFSET:[0-9a-f]+]]: CHECK-NEXT: [0x[[#%.16x,LOC_PAIR_START:]], 0x[[#%.16x,LOC_PAIR_END:]]): [[LOC_EXPR:.*]] CHECK-NEXT: [0x[[#%.16x,LOC_PAIR_START2:]], 0x[[#%.16x,LOC_PAIR_END2:]]): [[LOC_EXPR2:.*]]) CHECK: DW_AT_name [DW_FORM_strp] ( .debug_str[0x000000f2] = "x") @@ -105,7 +105,7 @@ CHECK-NEXT: (0x[[#sub(LOC_PAIR_START2,LOC_LOWPC)]], 0x[[#sub(LOC_PAIR CHECK: .debug_loclists contents: CHECK-NEXT: 0x00000000: locations list header: length = 0x00000018, format = DWARF32, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000 -CHECK-NEXT: 0x[[LOCLIST_OFFSET]]: +CHECK-NEXT: 0x[[LOCLIST_OFFSET]]: CHECK-NEXT: DW_LLE_base_addressx (0x0000000000000000) CHECK-NEXT: DW_LLE_offset_pair (0x[[#sub(LOCLIST_PAIR_START,LOCLIST_LOWPC)]], 0x[[#sub(LOCLIST_PAIR_END,LOCLIST_LOWPC)]]) CHECK-NEXT: DW_LLE_offset_pair (0x[[#sub(LOCLIST_PAIR_START2,LOCLIST_LOWPC)]], 0x[[#sub(LOCLIST_PAIR_END2,LOCLIST_LOWPC)]]) @@ -114,12 +114,12 @@ CHECK-NEXT: DW_LLE_end_of_list () CHECK: .debug_line contents: CHECK-NEXT: debug_line[0x00000000] CHECK-NEXT: Line table prologue: -CHECK-NEXT: total_length: 0x00000048 +CHECK-NEXT: total_length: 0x0000005a CHECK-NEXT: format: DWARF32 CHECK-NEXT: version: 5 CHECK-NEXT: address_size: 8 CHECK-NEXT: seg_select_size: 0 -CHECK-NEXT: prologue_length: 0x00000025 +CHECK-NEXT: prologue_length: 0x00000037 CHECK-NEXT: min_inst_length: 1 CHECK-NEXT: max_ops_per_inst: 1 CHECK-NEXT: default_is_stmt: 1 @@ -143,7 +143,7 @@ CHECK-NEXT: file_names[ 0]: CHECK-NEXT: name: .debug_line_str[0x00000029] = "a.cpp" CHECK-NEXT: dir_index: 0 -CHECK: debug_line[0x0000004c] +CHECK: debug_line[0x0000005e] CHECK-NEXT: Line table prologue: CHECK-NEXT: total_length: 0x0000003b CHECK-NEXT: format: DWARF32 diff --git a/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test b/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test index 13409b2a07a3..f6d42a18424c 100644 --- a/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test +++ b/llvm/test/tools/dsymutil/ARM/dwarf5-macho.test @@ -49,13 +49,13 @@ CHECK-NEXT: DW_AT_addr_base [DW_FORM_sec_offset] (0x00000008) CHECK: DW_TAG_subprogram CHECK-NEXT: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000000) address = 0x[[#%.16x,LOCLIST_LOWPC:]]) CHECK: DW_TAG_formal_parameter -CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] (0x[[LOC_OFFSET:[0-9a-f]+]]: +CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] (0x[[LOC_OFFSET:[0-9a-f]+]]: CHECK-NEXT: [0x[[#%.16x,LOCLIST_PAIR_START:]], 0x[[#%.16x,LOCLIST_PAIR_END:]]): [[LOCLIST_EXPR:.*]] CHECK-NEXT: [0x[[#%.16x,LOCLIST_PAIR_START2:]], 0x[[#%.16x,LOCLIST_PAIR_END2:]]): [[LOCLIST_EXPR2:.*]]) CHECK: .debug_loclists contents: CHECK-NEXT: 0x00000000: locations list header: length = 0x00000018, format = DWARF32, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000 -CHECK-NEXT: 0x[[LOC_OFFSET]]: +CHECK-NEXT: 0x[[LOC_OFFSET]]: CHECK-NEXT: DW_LLE_base_addressx (0x0000000000000000) CHECK-NEXT: DW_LLE_offset_pair (0x[[#sub(LOCLIST_PAIR_START,LOCLIST_LOWPC)]], 0x[[#sub(LOCLIST_PAIR_END,LOCLIST_LOWPC)]]) CHECK-NEXT: DW_LLE_offset_pair (0x[[#sub(LOCLIST_PAIR_START2,LOCLIST_LOWPC)]], 0x[[#sub(LOCLIST_PAIR_END2,LOCLIST_LOWPC)]]) @@ -64,12 +64,12 @@ CHECK-NEXT: DW_LLE_end_of_list () CHECK: .debug_line contents: CHECK-NEXT: debug_line[0x00000000] CHECK-NEXT: Line table prologue: -CHECK-NEXT: total_length: 0x00000048 +CHECK-NEXT: total_length: 0x0000005a CHECK-NEXT: format: DWARF32 CHECK-NEXT: version: 5 CHECK-NEXT: address_size: 8 CHECK-NEXT: seg_select_size: 0 -CHECK-NEXT: prologue_length: 0x00000025 +CHECK-NEXT: prologue_length: 0x00000037 CHECK-NEXT: min_inst_length: 1 CHECK-NEXT: max_ops_per_inst: 1 CHECK-NEXT: default_is_stmt: 1 @@ -92,6 +92,7 @@ CHECK-NEXT: include_directories[ 0] = .debug_line_str[0x00000000] = "/Users/sh CHECK-NEXT: file_names[ 0]: CHECK-NEXT: name: .debug_line_str[0x00000029] = "a.cpp" CHECK-NEXT: dir_index: 0 +CHECK-NEXT: md5_checksum: 2675ab7ce3623b564cfd8a2906a462e5 CHECK: .debug_str contents: -- GitLab From 2873060f3cfbd92dcff8d1037a08e9fb60f7882e Mon Sep 17 00:00:00 2001 From: Micah Weston Date: Fri, 5 Jan 2024 21:59:51 -0500 Subject: [PATCH 181/728] [SHT_LLVM_BB_ADDR_MAP] Fixes two bugs in decoding of PGOAnalyses in BBAddrMap. (#77139) We had specified that `readBBAddrMap` will always keep PGOAnalyses and BBAddrMaps the same length on success. https://github.com/llvm/llvm-project/blob/365fbbfbcfefb8766f7716109b9c3767b58e6058/llvm/include/llvm/Object/ELFObjectFile.h#L116-L117 It turns out that this is not currently the case when no analyses exist in a function. No test had caught it. We also should not append PGOBBEntries when there is no BBFreq or BrProb. This patch adds: * tests that PGOAnalyses and BBAddrMaps are same length even when no analyses are enabled * fixes decode so that PGOAnalyses and BBAddrMaps are same length * updates test to not emit unnecessary PGOBBEntries * fixes decode to not emit PGOBBEntries when unnecessary --- llvm/include/llvm/Object/ELFTypes.h | 3 ++ llvm/lib/Object/ELF.cpp | 7 ++-- llvm/unittests/Object/ELFObjectFileTest.cpp | 37 ++++++++++++++++----- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h index d3351a2d1650..956f7811dd6c 100644 --- a/llvm/include/llvm/Object/ELFTypes.h +++ b/llvm/include/llvm/Object/ELFTypes.h @@ -885,6 +885,9 @@ struct PGOAnalysisMap { bool BBFreq : 1; bool BrProb : 1; + // True if at least one feature is enabled + bool anyEnabled() const { return FuncEntryCount || BBFreq || BrProb; } + // Encodes to minimum bit width representation. uint8_t encode() const { return (static_cast(FuncEntryCount) << 0) | diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index 300639f2bfa0..f24395b02043 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -774,7 +774,7 @@ decodeBBAddrMapImpl(const ELFFile &EF, } FunctionEntries.emplace_back(Address, std::move(BBEntries)); - if (FeatEnable.FuncEntryCount || FeatEnable.BBFreq || FeatEnable.BrProb) { + if (PGOAnalyses || FeatEnable.anyEnabled()) { // Function entry count uint64_t FuncEntryCount = FeatEnable.FuncEntryCount @@ -782,8 +782,9 @@ decodeBBAddrMapImpl(const ELFFile &EF, : 0; std::vector PGOBBEntries; - for (uint32_t BlockIndex = 0; !MetadataDecodeErr && !ULEBSizeErr && Cur && - (BlockIndex < NumBlocks); + for (uint32_t BlockIndex = 0; + (FeatEnable.BBFreq || FeatEnable.BrProb) && !MetadataDecodeErr && + !ULEBSizeErr && Cur && (BlockIndex < NumBlocks); ++BlockIndex) { // Block frequency uint64_t BBF = FeatEnable.BBFreq diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp index 48bebc54ccc2..3a2a8e3d3264 100644 --- a/llvm/unittests/Object/ELFObjectFileTest.cpp +++ b/llvm/unittests/Object/ELFObjectFileTest.cpp @@ -1017,10 +1017,23 @@ Sections: BrProb: 0xffffffff - BBFreq: 1000 Successors: [] -)"); + - Name: .llvm_bb_addr_map_5 + Type: SHT_LLVM_BB_ADDR_MAP + # Link: 0 (by default, can be overriden) + Entries: + - Version: 2 + Address: 0x55555 + Feature: 0x0 + BBEntries: + - ID: 2 + AddressOffset: 0x0 + Size: 0x2 + Metadata: 0x4 + PGOAnalyses: [{}] + )"); BBAddrMap E1(0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}}}); - PGOAnalysisMap P1 = {892, {{}}, {true, false, false}}; + PGOAnalysisMap P1 = {892, {}, {true, false, false}}; BBAddrMap E2(0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}}}); PGOAnalysisMap P2 = {{}, {{BlockFrequency(343), {}}}, {false, true, false}}; BBAddrMap E3(0x33333, {{0, 0x0, 0x3, {false, true, true, false, false}}, @@ -1049,16 +1062,18 @@ Sections: {BlockFrequency(18), {{3, BranchProbability::getRaw(0xffff'ffff)}}}, {BlockFrequency(1000), {}}}, {true, true, true}}; + BBAddrMap E5(0x55555, {{2, 0x0, 0x2, {false, false, true, false, false}}}); + PGOAnalysisMap P5 = {{}, {}, {false, false, false}}; - std::vector Section0BBAddrMaps = {E4}; + std::vector Section0BBAddrMaps = {E4, E5}; std::vector Section1BBAddrMaps = {E3}; std::vector Section2BBAddrMaps = {E1, E2}; - std::vector AllBBAddrMaps = {E1, E2, E3, E4}; + std::vector AllBBAddrMaps = {E1, E2, E3, E4, E5}; - std::vector Section0PGOAnalysisMaps = {P4}; + std::vector Section0PGOAnalysisMaps = {P4, P5}; std::vector Section1PGOAnalysisMaps = {P3}; std::vector Section2PGOAnalysisMaps = {P1, P2}; - std::vector AllPGOAnalysisMaps = {P1, P2, P3, P4}; + std::vector AllPGOAnalysisMaps = {P1, P2, P3, P4, P5}; auto DoCheckSucceeds = [&](StringRef YamlString, std::optional TextSectionIndex, @@ -1081,6 +1096,10 @@ Sections: if (ExpectedPGO) { EXPECT_EQ(BBAddrMaps->size(), PGOAnalyses.size()); EXPECT_EQ(PGOAnalyses, *ExpectedPGO); + for (auto &&[BB, PGO] : llvm::zip(*BBAddrMaps, PGOAnalyses)) { + if (PGO.FeatEnable.BBFreq || PGO.FeatEnable.BrProb) + EXPECT_EQ(BB.getBBEntries().size(), PGO.BBEntries.size()); + } } }; @@ -1132,9 +1151,9 @@ Sections: Link: 10 )"; - DoCheckFails(InvalidLinkedYamlString, /*TextSectionIndex=*/4, + DoCheckFails(InvalidLinkedYamlString, /*TextSectionIndex=*/5, "unable to get the linked-to section for " - "SHT_LLVM_BB_ADDR_MAP section with index 4: invalid section " + "SHT_LLVM_BB_ADDR_MAP section with index 5: invalid section " "index: 10"); // Linked sections are not checked when we don't target a specific text // section. @@ -1150,7 +1169,7 @@ Sections: )"; DoCheckFails(TruncatedYamlString, /*TextSectionIndex=*/std::nullopt, - "unable to read SHT_LLVM_BB_ADDR_MAP section with index 4: " + "unable to read SHT_LLVM_BB_ADDR_MAP section with index 5: " "unable to decode LEB128 at offset 0x0000000a: malformed " "uleb128, extends past end"); // Check that we can read the other section's bb-address-maps which are -- GitLab From 1637c0792550f70e4b2ef42b3d08aa91dd27f4a9 Mon Sep 17 00:00:00 2001 From: Chaitanya Date: Sat, 6 Jan 2024 09:34:48 +0530 Subject: [PATCH 182/728] [openmp][amdgpu] Add DynamicLdsSize to AMDGPUImplicitArgsTy (#65325) #65273 "hidden_dynamic_lds_size" argument will be added in the reserved section at offset 120 of the implicit argument layout Add DynamicLdsSize to AMDGPUImplicitArgsTy struct at offset 120 and fill the dynamic LDS size before kernel launch. --- openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp | 1 + .../libomptarget/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp index 0411c6701334..18076f8082d0 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp @@ -3203,6 +3203,7 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice, ImplArgs->GroupSizeY = 1; ImplArgs->GroupSizeZ = 1; ImplArgs->GridDims = 1; + ImplArgs->DynamicLdsSize = KernelArgs.DynCGroupMem; } // Push the kernel launch into the stream. diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h b/openmp/libomptarget/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h index 2471590c27b3..58a3b5df00fa 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h @@ -45,7 +45,9 @@ struct AMDGPUImplicitArgsTy { uint16_t GroupSizeZ; uint8_t Unused0[46]; // 46 byte offset. uint16_t GridDims; - uint8_t Unused1[190]; // 190 byte offset. + uint8_t Unused1[54]; // 54 byte offset. + uint32_t DynamicLdsSize; + uint8_t Unused2[132]; // 132 byte offset. }; // Dummy struct for COV4 implicitargs. -- GitLab From ba3ef331b4568b5996172076572581e68c2d3c0c Mon Sep 17 00:00:00 2001 From: Mikhail Gudim Date: Fri, 5 Jan 2024 23:19:46 -0500 Subject: [PATCH 183/728] [RISCV][GlobalISel] Zbkb support for G_BSWAP (#77050) This instructions is legal in the presence of Zbkb extension. --- .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 2 +- .../instruction-select/bswap-rv32.mir | 15 ++--- .../instruction-select/bswap-rv64.mir | 15 ++--- .../legalizer/legalize-bswap-rv32.mir | 58 ++++++++++--------- .../legalizer/legalize-bswap-rv64.mir | 58 ++++++++++--------- 5 files changed, 77 insertions(+), 71 deletions(-) diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index 61bae5864925..ab8070772fe5 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -113,7 +113,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) getActionDefinitionsBuilder(G_BITREVERSE).maxScalar(0, sXLen).lower(); auto &BSWAPActions = getActionDefinitionsBuilder(G_BSWAP); - if (ST.hasStdExtZbb()) + if (ST.hasStdExtZbb() || ST.hasStdExtZbkb()) BSWAPActions.legalFor({sXLen}).clampScalar(0, sXLen, sXLen); else BSWAPActions.maxScalar(0, sXLen).lower(); diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv32.mir index 733fd128282e..721721cf361e 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv32.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv32.mir @@ -1,7 +1,8 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=riscv32 -mattr=+zbb -run-pass=instruction-select \ -# RUN: -simplify-mir -verify-machineinstrs %s -o - \ -# RUN: | FileCheck -check-prefix=RV32I %s +# RUN: -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple=riscv32 -mattr=+zbkb -run-pass=instruction-select \ +# RUN: -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s --- name: bswap_s32 @@ -9,11 +10,11 @@ legalized: true regBankSelected: true body: | bb.0.entry: - ; RV32I-LABEL: name: bswap_s32 - ; RV32I: [[COPY:%[0-9]+]]:gpr = COPY $x10 - ; RV32I-NEXT: [[REV8_RV32_:%[0-9]+]]:gpr = REV8_RV32 [[COPY]] - ; RV32I-NEXT: $x10 = COPY [[REV8_RV32_]] - ; RV32I-NEXT: PseudoRET implicit $x10 + ; CHECK-LABEL: name: bswap_s32 + ; CHECK: [[COPY:%[0-9]+]]:gpr = COPY $x10 + ; CHECK-NEXT: [[REV8_RV32_:%[0-9]+]]:gpr = REV8_RV32 [[COPY]] + ; CHECK-NEXT: $x10 = COPY [[REV8_RV32_]] + ; CHECK-NEXT: PseudoRET implicit $x10 %0:gprb(s32) = COPY $x10 %1:gprb(s32) = G_BSWAP %0 $x10 = COPY %1(s32) diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv64.mir index 053abef93a3a..6cdfb76f0b47 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv64.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/bswap-rv64.mir @@ -1,7 +1,8 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py # RUN: llc -mtriple=riscv64 -mattr=+zbb -run-pass=instruction-select \ -# RUN: -simplify-mir -verify-machineinstrs %s -o - \ -# RUN: | FileCheck -check-prefix=RV64I %s +# RUN: -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple=riscv64 -mattr=+zbkb -run-pass=instruction-select \ +# RUN: -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s --- name: bswap_s64 @@ -9,11 +10,11 @@ legalized: true regBankSelected: true body: | bb.0.entry: - ; RV64I-LABEL: name: bswap_s64 - ; RV64I: [[COPY:%[0-9]+]]:gpr = COPY $x10 - ; RV64I-NEXT: [[REV8_RV64_:%[0-9]+]]:gpr = REV8_RV64 [[COPY]] - ; RV64I-NEXT: $x10 = COPY [[REV8_RV64_]] - ; RV64I-NEXT: PseudoRET implicit $x10 + ; CHECK-LABEL: name: bswap_s64 + ; CHECK: [[COPY:%[0-9]+]]:gpr = COPY $x10 + ; CHECK-NEXT: [[REV8_RV64_:%[0-9]+]]:gpr = REV8_RV64 [[COPY]] + ; CHECK-NEXT: $x10 = COPY [[REV8_RV64_]] + ; CHECK-NEXT: PseudoRET implicit $x10 %0:gprb(s64) = COPY $x10 %1:gprb(s64) = G_BSWAP %0 $x10 = COPY %1(s64) diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir index e66dbfa4c821..d6598c89b39a 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv32.mir @@ -2,7 +2,9 @@ # RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - \ # RUN: | FileCheck %s --check-prefix=RV32I # RUN: llc -mtriple=riscv32 -mattr=+zbb -run-pass=legalizer %s -o - \ -# RUN: | FileCheck %s --check-prefix=RV32ZBB +# RUN: | FileCheck %s --check-prefix=RV32ZBB_OR_RV32ZBKB +# RUN: llc -mtriple=riscv32 -mattr=+zbkb -run-pass=legalizer %s -o - \ +# RUN: | FileCheck %s --check-prefix=RV32ZBB_OR_RV32ZBKB --- name: bswap_i16 @@ -23,16 +25,16 @@ body: | ; RV32I-NEXT: $x10 = COPY [[AND]](s32) ; RV32I-NEXT: PseudoRET implicit $x10 ; - ; RV32ZBB-LABEL: name: bswap_i16 - ; RV32ZBB: liveins: $x10 - ; RV32ZBB-NEXT: {{ $}} - ; RV32ZBB-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 - ; RV32ZBB-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[COPY]], 16 - ; RV32ZBB-NEXT: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[ASSERT_ZEXT]] - ; RV32ZBB-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16 - ; RV32ZBB-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP]], [[C]](s32) - ; RV32ZBB-NEXT: $x10 = COPY [[LSHR]](s32) - ; RV32ZBB-NEXT: PseudoRET implicit $x10 + ; RV32ZBB_OR_RV32ZBKB-LABEL: name: bswap_i16 + ; RV32ZBB_OR_RV32ZBKB: liveins: $x10 + ; RV32ZBB_OR_RV32ZBKB-NEXT: {{ $}} + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[COPY]], 16 + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[ASSERT_ZEXT]] + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16 + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP]], [[C]](s32) + ; RV32ZBB_OR_RV32ZBKB-NEXT: $x10 = COPY [[LSHR]](s32) + ; RV32ZBB_OR_RV32ZBKB-NEXT: PseudoRET implicit $x10 %0:_(s32) = COPY $x10 %1:_(s32) = G_ASSERT_ZEXT %0, 16 %2:_(s16) = G_TRUNC %1(s32) @@ -65,13 +67,13 @@ body: | ; RV32I-NEXT: $x10 = COPY [[OR2]](s32) ; RV32I-NEXT: PseudoRET implicit $x10 ; - ; RV32ZBB-LABEL: name: bswap_i32 - ; RV32ZBB: liveins: $x10 - ; RV32ZBB-NEXT: {{ $}} - ; RV32ZBB-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 - ; RV32ZBB-NEXT: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY]] - ; RV32ZBB-NEXT: $x10 = COPY [[BSWAP]](s32) - ; RV32ZBB-NEXT: PseudoRET implicit $x10 + ; RV32ZBB_OR_RV32ZBKB-LABEL: name: bswap_i32 + ; RV32ZBB_OR_RV32ZBKB: liveins: $x10 + ; RV32ZBB_OR_RV32ZBKB-NEXT: {{ $}} + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY]] + ; RV32ZBB_OR_RV32ZBKB-NEXT: $x10 = COPY [[BSWAP]](s32) + ; RV32ZBB_OR_RV32ZBKB-NEXT: PseudoRET implicit $x10 %0:_(s32) = COPY $x10 %1:_(s32) = G_BSWAP %0 $x10 = COPY %1(s32) @@ -115,16 +117,16 @@ body: | ; RV32I-NEXT: $x11 = COPY [[OR5]](s32) ; RV32I-NEXT: PseudoRET implicit $x10, implicit $x11 ; - ; RV32ZBB-LABEL: name: bswap_i64 - ; RV32ZBB: liveins: $x10, $x11 - ; RV32ZBB-NEXT: {{ $}} - ; RV32ZBB-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 - ; RV32ZBB-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 - ; RV32ZBB-NEXT: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY1]] - ; RV32ZBB-NEXT: [[BSWAP1:%[0-9]+]]:_(s32) = G_BSWAP [[COPY]] - ; RV32ZBB-NEXT: $x10 = COPY [[BSWAP]](s32) - ; RV32ZBB-NEXT: $x11 = COPY [[BSWAP1]](s32) - ; RV32ZBB-NEXT: PseudoRET implicit $x10, implicit $x11 + ; RV32ZBB_OR_RV32ZBKB-LABEL: name: bswap_i64 + ; RV32ZBB_OR_RV32ZBKB: liveins: $x10, $x11 + ; RV32ZBB_OR_RV32ZBKB-NEXT: {{ $}} + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY1]] + ; RV32ZBB_OR_RV32ZBKB-NEXT: [[BSWAP1:%[0-9]+]]:_(s32) = G_BSWAP [[COPY]] + ; RV32ZBB_OR_RV32ZBKB-NEXT: $x10 = COPY [[BSWAP]](s32) + ; RV32ZBB_OR_RV32ZBKB-NEXT: $x11 = COPY [[BSWAP1]](s32) + ; RV32ZBB_OR_RV32ZBKB-NEXT: PseudoRET implicit $x10, implicit $x11 %0:_(s32) = COPY $x10 %1:_(s32) = COPY $x11 %2:_(s64) = G_MERGE_VALUES %0(s32), %1(s32) diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir index b73a22c1a07e..61a0de973984 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-bswap-rv64.mir @@ -2,7 +2,9 @@ # RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - \ # RUN: | FileCheck %s --check-prefix=RV64I # RUN: llc -mtriple=riscv64 -mattr=+zbb -run-pass=legalizer %s -o - \ -# RUN: | FileCheck %s --check-prefix=RV64ZBB +# RUN: | FileCheck %s --check-prefix=RV64ZBB_OR_RV64ZBKB +# RUN: llc -mtriple=riscv64 -mattr=+zbkb -run-pass=legalizer %s -o - \ +# RUN: | FileCheck %s --check-prefix=RV64ZBB_OR_RV64ZBKB --- name: bswap_i16 @@ -27,16 +29,16 @@ body: | ; RV64I-NEXT: $x10 = COPY [[AND]](s64) ; RV64I-NEXT: PseudoRET implicit $x10 ; - ; RV64ZBB-LABEL: name: bswap_i16 - ; RV64ZBB: liveins: $x10 - ; RV64ZBB-NEXT: {{ $}} - ; RV64ZBB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 - ; RV64ZBB-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s64) = G_ASSERT_ZEXT [[COPY]], 16 - ; RV64ZBB-NEXT: [[BSWAP:%[0-9]+]]:_(s64) = G_BSWAP [[ASSERT_ZEXT]] - ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48 - ; RV64ZBB-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[BSWAP]], [[C]](s64) - ; RV64ZBB-NEXT: $x10 = COPY [[LSHR]](s64) - ; RV64ZBB-NEXT: PseudoRET implicit $x10 + ; RV64ZBB_OR_RV64ZBKB-LABEL: name: bswap_i16 + ; RV64ZBB_OR_RV64ZBKB: liveins: $x10 + ; RV64ZBB_OR_RV64ZBKB-NEXT: {{ $}} + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s64) = G_ASSERT_ZEXT [[COPY]], 16 + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[BSWAP:%[0-9]+]]:_(s64) = G_BSWAP [[ASSERT_ZEXT]] + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48 + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[BSWAP]], [[C]](s64) + ; RV64ZBB_OR_RV64ZBKB-NEXT: $x10 = COPY [[LSHR]](s64) + ; RV64ZBB_OR_RV64ZBKB-NEXT: PseudoRET implicit $x10 %0:_(s64) = COPY $x10 %1:_(s64) = G_ASSERT_ZEXT %0, 16 %2:_(s16) = G_TRUNC %1(s64) @@ -74,16 +76,16 @@ body: | ; RV64I-NEXT: $x10 = COPY [[ZEXT]](s64) ; RV64I-NEXT: PseudoRET implicit $x10 ; - ; RV64ZBB-LABEL: name: bswap_i32 - ; RV64ZBB: liveins: $x10 - ; RV64ZBB-NEXT: {{ $}} - ; RV64ZBB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 - ; RV64ZBB-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s64) = G_ASSERT_ZEXT [[COPY]], 32 - ; RV64ZBB-NEXT: [[BSWAP:%[0-9]+]]:_(s64) = G_BSWAP [[ASSERT_ZEXT]] - ; RV64ZBB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32 - ; RV64ZBB-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[BSWAP]], [[C]](s64) - ; RV64ZBB-NEXT: $x10 = COPY [[LSHR]](s64) - ; RV64ZBB-NEXT: PseudoRET implicit $x10 + ; RV64ZBB_OR_RV64ZBKB-LABEL: name: bswap_i32 + ; RV64ZBB_OR_RV64ZBKB: liveins: $x10 + ; RV64ZBB_OR_RV64ZBKB-NEXT: {{ $}} + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s64) = G_ASSERT_ZEXT [[COPY]], 32 + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[BSWAP:%[0-9]+]]:_(s64) = G_BSWAP [[ASSERT_ZEXT]] + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32 + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[BSWAP]], [[C]](s64) + ; RV64ZBB_OR_RV64ZBKB-NEXT: $x10 = COPY [[LSHR]](s64) + ; RV64ZBB_OR_RV64ZBKB-NEXT: PseudoRET implicit $x10 %0:_(s64) = COPY $x10 %1:_(s64) = G_ASSERT_ZEXT %0, 32 %2:_(s32) = G_TRUNC %1(s64) @@ -132,13 +134,13 @@ body: | ; RV64I-NEXT: $x10 = COPY [[OR6]](s64) ; RV64I-NEXT: PseudoRET implicit $x10 ; - ; RV64ZBB-LABEL: name: bswap_i64 - ; RV64ZBB: liveins: $x10 - ; RV64ZBB-NEXT: {{ $}} - ; RV64ZBB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 - ; RV64ZBB-NEXT: [[BSWAP:%[0-9]+]]:_(s64) = G_BSWAP [[COPY]] - ; RV64ZBB-NEXT: $x10 = COPY [[BSWAP]](s64) - ; RV64ZBB-NEXT: PseudoRET implicit $x10 + ; RV64ZBB_OR_RV64ZBKB-LABEL: name: bswap_i64 + ; RV64ZBB_OR_RV64ZBKB: liveins: $x10 + ; RV64ZBB_OR_RV64ZBKB-NEXT: {{ $}} + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64ZBB_OR_RV64ZBKB-NEXT: [[BSWAP:%[0-9]+]]:_(s64) = G_BSWAP [[COPY]] + ; RV64ZBB_OR_RV64ZBKB-NEXT: $x10 = COPY [[BSWAP]](s64) + ; RV64ZBB_OR_RV64ZBKB-NEXT: PseudoRET implicit $x10 %0:_(s64) = COPY $x10 %1:_(s64) = G_BSWAP %0 $x10 = COPY %1(s64) -- GitLab From a5902a4d2425ac083f1530719e35b5c562cb1e60 Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Sat, 6 Jan 2024 11:33:36 +0800 Subject: [PATCH 184/728] [X86][NFC] Rename variables/passes for EVEX compression optimization RFC: https://discourse.llvm.org/t/rfc-design-for-apx-feature-egpr-and-ndd-support/73031 APX introduces EGPR, NDD and NF instructions. In addition to compressing EVEX encoded AVX512 instructions into VEX encoding, we also have several more possible optimizations. a. Promoted instruction (EVEX space) -> pre-promotion instruction (legacy space) b. NDD (EVEX space) -> non-NDD (legacy space) c. NF_ND (EVEX space) -> NF (EVEX space) The first two types of compression can usually reduce code size, while the third type of compression can help hardware decode although the instruction length remains unchanged. So we do the renaming for the upcoming APX optimizations. BTW, I clang-format the code in X86CompressEVEX.cpp, X86CompressEVEXTablesEmitter.cpp. This patch also extracts the NFC in #77065 into a separate commit. --- llvm/lib/Target/X86/CMakeLists.txt | 4 +- llvm/lib/Target/X86/X86.h | 8 +- .../{X86EvexToVex.cpp => X86CompressEVEX.cpp} | 113 ++++++++++-------- llvm/lib/Target/X86/X86TargetMachine.cpp | 4 +- llvm/test/CodeGen/X86/O0-pipeline.ll | 2 +- .../test/CodeGen/X86/evex-to-vex-compress.mir | 2 +- llvm/test/CodeGen/X86/opt-pipeline.ll | 2 +- llvm/utils/TableGen/CMakeLists.txt | 2 +- ...r.cpp => X86CompressEVEXTablesEmitter.cpp} | 61 +++++----- 9 files changed, 105 insertions(+), 93 deletions(-) rename llvm/lib/Target/X86/{X86EvexToVex.cpp => X86CompressEVEX.cpp} (71%) rename llvm/utils/TableGen/{X86EVEX2VEXTablesEmitter.cpp => X86CompressEVEXTablesEmitter.cpp} (77%) diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt index 0b7a98ad6341..4d6300cad2a7 100644 --- a/llvm/lib/Target/X86/CMakeLists.txt +++ b/llvm/lib/Target/X86/CMakeLists.txt @@ -8,7 +8,7 @@ tablegen(LLVM X86GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1) tablegen(LLVM X86GenCallingConv.inc -gen-callingconv) tablegen(LLVM X86GenDAGISel.inc -gen-dag-isel) tablegen(LLVM X86GenDisassemblerTables.inc -gen-disassembler) -tablegen(LLVM X86GenEVEX2VEXTables.inc -gen-x86-EVEX2VEX-tables) +tablegen(LLVM X86GenCompressEVEXTables.inc -gen-x86-compress-evex-tables) tablegen(LLVM X86GenExegesis.inc -gen-exegesis) tablegen(LLVM X86GenFastISel.inc -gen-fast-isel) tablegen(LLVM X86GenGlobalISel.inc -gen-global-isel) @@ -61,7 +61,7 @@ set(sources X86InstrFMA3Info.cpp X86InstrFoldTables.cpp X86InstrInfo.cpp - X86EvexToVex.cpp + X86CompressEVEX.cpp X86LoadValueInjectionLoadHardening.cpp X86LoadValueInjectionRetHardening.cpp X86MCInstLower.cpp diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h index 485afbc1dfbc..21623a805f55 100644 --- a/llvm/lib/Target/X86/X86.h +++ b/llvm/lib/Target/X86/X86.h @@ -131,9 +131,9 @@ FunctionPass *createX86FixupBWInsts(); /// to another, when profitable. FunctionPass *createX86DomainReassignmentPass(); -/// This pass replaces EVEX encoded of AVX-512 instructiosn by VEX -/// encoding when possible in order to reduce code size. -FunctionPass *createX86EvexToVexInsts(); +/// This pass compress instructions from EVEX space to legacy/VEX/EVEX space when +/// possible in order to reduce code size or facilitate HW decoding. +FunctionPass *createX86CompressEVEXPass(); /// This pass creates the thunks for the retpoline feature. FunctionPass *createX86IndirectThunksPass(); @@ -167,7 +167,7 @@ FunctionPass *createX86SpeculativeLoadHardeningPass(); FunctionPass *createX86SpeculativeExecutionSideEffectSuppression(); FunctionPass *createX86ArgumentStackSlotPass(); -void initializeEvexToVexInstPassPass(PassRegistry &); +void initializeCompressEVEXPassPass(PassRegistry &); void initializeFPSPass(PassRegistry &); void initializeFixupBWInstPassPass(PassRegistry &); void initializeFixupLEAPassPass(PassRegistry &); diff --git a/llvm/lib/Target/X86/X86EvexToVex.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp similarity index 71% rename from llvm/lib/Target/X86/X86EvexToVex.cpp rename to llvm/lib/Target/X86/X86CompressEVEX.cpp index c425c37b4186..accb98cec19d 100644 --- a/llvm/lib/Target/X86/X86EvexToVex.cpp +++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp @@ -1,5 +1,4 @@ -//===- X86EvexToVex.cpp ---------------------------------------------------===// -// Compress EVEX instructions to VEX encoding when possible to reduce code size +//===- X86CompressEVEX.cpp ------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -7,17 +6,34 @@ // //===----------------------------------------------------------------------===// // -/// \file -/// This file defines the pass that goes over all AVX-512 instructions which -/// are encoded using the EVEX prefix and if possible replaces them by their -/// corresponding VEX encoding which is usually shorter by 2 bytes. -/// EVEX instructions may be encoded via the VEX prefix when the AVX-512 -/// instruction has a corresponding AVX/AVX2 opcode, when vector length -/// accessed by instruction is less than 512 bits and when it does not use -// the xmm or the mask registers or xmm/ymm registers with indexes higher -// than 15. -/// The pass applies code reduction on the generated code for AVX-512 instrs. +// This pass compresses instructions from EVEX space to legacy/VEX/EVEX space +// when possible in order to reduce code size or facilitate HW decoding. // +// Possible compression: +// a. AVX512 instruction (EVEX) -> AVX instruction (VEX) +// b. Promoted instruction (EVEX) -> pre-promotion instruction (legacy) +// c. NDD (EVEX) -> non-NDD (legacy) +// d. NF_ND (EVEX) -> NF (EVEX) +// +// Compression a, b and c always reduce code size (some exception) +// fourth type of compression can help hardware decode although the instruction +// length remains unchanged. +// +// Compression a, b and c can always reduce code size, with some exceptions +// such as promoted 16-bit CRC32 which is as long as the legacy version. +// +// legacy: +// crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6] +// promoted: +// crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6] +// +// From performance perspective, these should be same (same uops and same EXE +// ports). From a FMV perspective, an older legacy encoding is preferred b/c it +// can execute in more places (broader HW install base). So we will still do +// the compression. +// +// Compression d can help hardware decode (HW may skip reading the NDD +// register) although the instruction length remains unchanged. //===----------------------------------------------------------------------===// #include "MCTargetDesc/X86BaseInfo.h" @@ -38,37 +54,34 @@ using namespace llvm; -// Including the generated EVEX2VEX tables. -struct X86EvexToVexCompressTableEntry { - uint16_t EvexOpc; - uint16_t VexOpc; +// Including the generated EVEX compression tables. +struct X86CompressEVEXTableEntry { + uint16_t OldOpc; + uint16_t NewOpc; - bool operator<(const X86EvexToVexCompressTableEntry &RHS) const { - return EvexOpc < RHS.EvexOpc; + bool operator<(const X86CompressEVEXTableEntry &RHS) const { + return OldOpc < RHS.OldOpc; } - friend bool operator<(const X86EvexToVexCompressTableEntry &TE, - unsigned Opc) { - return TE.EvexOpc < Opc; + friend bool operator<(const X86CompressEVEXTableEntry &TE, unsigned Opc) { + return TE.OldOpc < Opc; } }; -#include "X86GenEVEX2VEXTables.inc" +#include "X86GenCompressEVEXTables.inc" -#define EVEX2VEX_DESC "Compressing EVEX instrs to VEX encoding when possible" -#define EVEX2VEX_NAME "x86-evex-to-vex-compress" +#define COMP_EVEX_DESC "Compressing EVEX instrs when possible" +#define COMP_EVEX_NAME "x86-compress-evex" -#define DEBUG_TYPE EVEX2VEX_NAME +#define DEBUG_TYPE COMP_EVEX_NAME namespace { -class EvexToVexInstPass : public MachineFunctionPass { +class CompressEVEXPass : public MachineFunctionPass { public: static char ID; - EvexToVexInstPass() : MachineFunctionPass(ID) {} - StringRef getPassName() const override { return EVEX2VEX_DESC; } + CompressEVEXPass() : MachineFunctionPass(ID) {} + StringRef getPassName() const override { return COMP_EVEX_DESC; } - /// Loop over all of the basic blocks, replacing EVEX instructions - /// by equivalent VEX instructions when possible for reducing code size. bool runOnMachineFunction(MachineFunction &MF) override; // This pass runs after regalloc and doesn't support VReg operands. @@ -80,7 +93,7 @@ public: } // end anonymous namespace -char EvexToVexInstPass::ID = 0; +char CompressEVEXPass::ID = 0; static bool usesExtendedRegister(const MachineInstr &MI) { auto isHiRegIdx = [](unsigned Reg) { @@ -112,8 +125,8 @@ static bool usesExtendedRegister(const MachineInstr &MI) { return false; } -static bool checkVEXInstPredicate(unsigned EvexOpc, const X86Subtarget &ST) { - switch (EvexOpc) { +static bool checkVEXInstPredicate(unsigned OldOpc, const X86Subtarget &ST) { + switch (OldOpc) { default: return true; case X86::VCVTNEPS2BF16Z128rm: @@ -151,15 +164,15 @@ static bool checkVEXInstPredicate(unsigned EvexOpc, const X86Subtarget &ST) { } // Do any custom cleanup needed to finalize the conversion. -static bool performCustomAdjustments(MachineInstr &MI, unsigned VexOpc) { - (void)VexOpc; +static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) { + (void)NewOpc; unsigned Opc = MI.getOpcode(); switch (Opc) { case X86::VALIGNDZ128rri: case X86::VALIGNDZ128rmi: case X86::VALIGNQZ128rri: case X86::VALIGNQZ128rmi: { - assert((VexOpc == X86::VPALIGNRrri || VexOpc == X86::VPALIGNRrmi) && + assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) && "Unexpected new opcode!"); unsigned Scale = (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4; @@ -175,8 +188,8 @@ static bool performCustomAdjustments(MachineInstr &MI, unsigned VexOpc) { case X86::VSHUFI32X4Z256rri: case X86::VSHUFI64X2Z256rmi: case X86::VSHUFI64X2Z256rri: { - assert((VexOpc == X86::VPERM2F128rr || VexOpc == X86::VPERM2I128rr || - VexOpc == X86::VPERM2F128rm || VexOpc == X86::VPERM2I128rm) && + assert((NewOpc == X86::VPERM2F128rr || NewOpc == X86::VPERM2I128rr || + NewOpc == X86::VPERM2F128rm || NewOpc == X86::VPERM2I128rm) && "Unexpected new opcode!"); MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1); int64_t ImmVal = Imm.getImm(); @@ -200,7 +213,7 @@ static bool performCustomAdjustments(MachineInstr &MI, unsigned VexOpc) { case X86::VRNDSCALESDZm_Int: case X86::VRNDSCALESSZr_Int: case X86::VRNDSCALESSZm_Int: - const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1); + const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1); int64_t ImmVal = Imm.getImm(); // Ensure that only bits 3:0 of the immediate are used. if ((ImmVal & 0xf) != ImmVal) @@ -239,28 +252,28 @@ static bool CompressEvexToVexImpl(MachineInstr &MI, const X86Subtarget &ST) { return false; // Use the VEX.L bit to select the 128 or 256-bit table. - ArrayRef Table = + ArrayRef Table = (Desc.TSFlags & X86II::VEX_L) ? ArrayRef(X86EvexToVex256CompressTable) : ArrayRef(X86EvexToVex128CompressTable); - unsigned EvexOpc = MI.getOpcode(); - const auto *I = llvm::lower_bound(Table, EvexOpc); - if (I == Table.end() || I->EvexOpc != EvexOpc) + unsigned Opc = MI.getOpcode(); + const auto *I = llvm::lower_bound(Table, Opc); + if (I == Table.end() || I->OldOpc != Opc) return false; if (usesExtendedRegister(MI)) return false; - if (!checkVEXInstPredicate(EvexOpc, ST)) + if (!checkVEXInstPredicate(Opc, ST)) return false; - if (!performCustomAdjustments(MI, I->VexOpc)) + if (!performCustomAdjustments(MI, I->NewOpc)) return false; - MI.setDesc(ST.getInstrInfo()->get(I->VexOpc)); + MI.setDesc(ST.getInstrInfo()->get(I->NewOpc)); MI.setAsmPrinterFlag(X86::AC_EVEX_2_VEX); return true; } -bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) { +bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) { #ifndef NDEBUG // Make sure the tables are sorted. static std::atomic TableChecked(false); @@ -289,8 +302,8 @@ bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) { return Changed; } -INITIALIZE_PASS(EvexToVexInstPass, EVEX2VEX_NAME, EVEX2VEX_DESC, false, false) +INITIALIZE_PASS(CompressEVEXPass, COMP_EVEX_NAME, COMP_EVEX_DESC, false, false) -FunctionPass *llvm::createX86EvexToVexInsts() { - return new EvexToVexInstPass(); +FunctionPass *llvm::createX86CompressEVEXPass() { + return new CompressEVEXPass(); } diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 5668b514d6de..b92bffbe6239 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -75,7 +75,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() { initializeGlobalISel(PR); initializeWinEHStatePassPass(PR); initializeFixupBWInstPassPass(PR); - initializeEvexToVexInstPassPass(PR); + initializeCompressEVEXPassPass(PR); initializeFixupLEAPassPass(PR); initializeFPSPass(PR); initializeX86FixupSetCCPassPass(PR); @@ -575,7 +575,7 @@ void X86PassConfig::addPreEmitPass() { addPass(createX86FixupInstTuning()); addPass(createX86FixupVectorConstants()); } - addPass(createX86EvexToVexInsts()); + addPass(createX86CompressEVEXPass()); addPass(createX86DiscriminateMemOpsPass()); addPass(createX86InsertPrefetchPass()); addPass(createX86InsertX87waitPass()); diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll index 402645ed1e2e..11025b0e6bf2 100644 --- a/llvm/test/CodeGen/X86/O0-pipeline.ll +++ b/llvm/test/CodeGen/X86/O0-pipeline.ll @@ -68,7 +68,7 @@ ; CHECK-NEXT: Implement the 'patchable-function' attribute ; CHECK-NEXT: X86 Indirect Branch Tracking ; CHECK-NEXT: X86 vzeroupper inserter -; CHECK-NEXT: Compressing EVEX instrs to VEX encoding when possibl +; CHECK-NEXT: Compressing EVEX instrs when possible ; CHECK-NEXT: X86 Discriminate Memory Operands ; CHECK-NEXT: X86 Insert Cache Prefetches ; CHECK-NEXT: X86 insert wait instruction diff --git a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir index 06d3c1532c3e..548cf24b9200 100644 --- a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir +++ b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir @@ -1,4 +1,4 @@ -# RUN: llc -mtriple=x86_64-- -run-pass x86-evex-to-vex-compress -verify-machineinstrs -mcpu=skx -o - %s | FileCheck %s +# RUN: llc -mtriple=x86_64-- -run-pass x86-compress-evex -verify-machineinstrs -mcpu=skx -o - %s | FileCheck %s # This test verifies VEX encoding for AVX-512 instructions that use registers of low indexes and # do not use zmm or mask registers and have a corresponding AVX/AVX2 opcode diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll index fb8d2335b341..6f2bba84a6ec 100644 --- a/llvm/test/CodeGen/X86/opt-pipeline.ll +++ b/llvm/test/CodeGen/X86/opt-pipeline.ll @@ -205,7 +205,7 @@ ; CHECK-NEXT: X86 LEA Fixup ; CHECK-NEXT: X86 Fixup Inst Tuning ; CHECK-NEXT: X86 Fixup Vector Constants -; CHECK-NEXT: Compressing EVEX instrs to VEX encoding when possible +; CHECK-NEXT: Compressing EVEX instrs when possible ; CHECK-NEXT: X86 Discriminate Memory Operands ; CHECK-NEXT: X86 Insert Cache Prefetches ; CHECK-NEXT: X86 insert wait instruction diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt index f765cc36d3be..0100bf345ec2 100644 --- a/llvm/utils/TableGen/CMakeLists.txt +++ b/llvm/utils/TableGen/CMakeLists.txt @@ -82,7 +82,7 @@ add_tablegen(llvm-tblgen LLVM Types.cpp VarLenCodeEmitterGen.cpp X86DisassemblerTables.cpp - X86EVEX2VEXTablesEmitter.cpp + X86CompressEVEXTablesEmitter.cpp X86FoldTablesEmitter.cpp X86MnemonicTables.cpp X86ModRMFilters.cpp diff --git a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp similarity index 77% rename from llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp rename to llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index c80d9a199fa3..c1ea34dc02e2 100644 --- a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -1,4 +1,4 @@ -//===- utils/TableGen/X86EVEX2VEXTablesEmitter.cpp - X86 backend-*- C++ -*-===// +//==- utils/TableGen/X86CompressEVEXTablesEmitter.cpp - X86 backend-*- C++ -*-// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// /// -/// This tablegen backend is responsible for emitting the X86 backend EVEX2VEX +/// This tablegen backend is responsible for emitting the X86 backend EVEX /// compression tables. /// //===----------------------------------------------------------------------===// @@ -23,15 +23,15 @@ using namespace X86Disassembler; namespace { -class X86EVEX2VEXTablesEmitter { +class X86CompressEVEXTablesEmitter { RecordKeeper &Records; CodeGenTarget Target; - // Hold all non-masked & non-broadcasted EVEX encoded instructions - std::vector EVEXInsts; - // Hold all VEX encoded instructions. Divided into groups with same opcodes + // Hold all pontentially compressible EVEX instructions + std::vector PreCompressionInsts; + // Hold all compressed instructions. Divided into groups with same opcodes // to make the search more efficient - std::map> VEXInsts; + std::map> CompressedInsts; typedef std::pair Entry; @@ -41,25 +41,24 @@ class X86EVEX2VEXTablesEmitter { std::vector EVEX2VEX256; public: - X86EVEX2VEXTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} + X86CompressEVEXTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} - // run - Output X86 EVEX2VEX tables. + // run - Output X86 EVEX compression tables. void run(raw_ostream &OS); private: - // Prints the given table as a C++ array of type - // X86EvexToVexCompressTableEntry + // Prints the given table as a C++ array of type X86CompressEVEXTableEntry void printTable(const std::vector &Table, raw_ostream &OS); }; -void X86EVEX2VEXTablesEmitter::printTable(const std::vector &Table, - raw_ostream &OS) { +void X86CompressEVEXTablesEmitter::printTable(const std::vector &Table, + raw_ostream &OS) { StringRef Size = (Table == EVEX2VEX128) ? "128" : "256"; OS << "// X86 EVEX encoded instructions that have a VEX " << Size << " encoding\n" << "// (table format: ).\n" - << "static const X86EvexToVexCompressTableEntry X86EvexToVex" << Size + << "static const X86CompressEVEXTableEntry X86EvexToVex" << Size << "CompressTable[] = {\n" << " // EVEX scalar with corresponding VEX.\n"; @@ -98,8 +97,8 @@ public: RecognizableInstrBase EVEXRI(*EVEXInst); bool VEX_W = VEXRI.HasREX_W; bool EVEX_W = EVEXRI.HasREX_W; - bool VEX_WIG = VEXRI.IgnoresW; - bool EVEX_WIG = EVEXRI.IgnoresW; + bool VEX_WIG = VEXRI.IgnoresW; + bool EVEX_WIG = EVEXRI.IgnoresW; bool EVEX_W1_VEX_W0 = EVEXInst->TheDef->getValueAsBit("EVEX_W1_VEX_W0"); if (VEXRI.IsCodeGenOnly != EVEXRI.IsCodeGenOnly || @@ -145,8 +144,8 @@ public: } }; -void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { - emitSourceFileHeader("X86 EVEX2VEX tables", OS); +void X86CompressEVEXTablesEmitter::run(raw_ostream &OS) { + emitSourceFileHeader("X86 EVEX compression tables", OS); ArrayRef NumberedInstructions = Target.getInstructionsByEnumValue(); @@ -161,32 +160,32 @@ void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { continue; RecognizableInstrBase RI(*Inst); - // Add VEX encoded instructions to one of VEXInsts vectors according to - // it's opcode. + // Add VEX encoded instructions to one of CompressedInsts vectors according + // to it's opcode. if (RI.Encoding == X86Local::VEX) - VEXInsts[RI.Opcode].push_back(Inst); - // Add relevant EVEX encoded instructions to EVEXInsts + CompressedInsts[RI.Opcode].push_back(Inst); + // Add relevant EVEX encoded instructions to PreCompressionInsts else if (RI.Encoding == X86Local::EVEX && !RI.HasEVEX_K && !RI.HasEVEX_B && !RI.HasEVEX_L2 && !Def->getValueAsBit("notEVEX2VEXConvertible")) - EVEXInsts.push_back(Inst); + PreCompressionInsts.push_back(Inst); } - for (const CodeGenInstruction *EVEXInst : EVEXInsts) { - uint64_t Opcode = getValueFromBitsInit(EVEXInst->TheDef-> - getValueAsBitsInit("Opcode")); + for (const CodeGenInstruction *EVEXInst : PreCompressionInsts) { + uint64_t Opcode = + getValueFromBitsInit(EVEXInst->TheDef->getValueAsBitsInit("Opcode")); // For each EVEX instruction look for a VEX match in the appropriate vector // (instructions with the same opcode) using function object IsMatch. // Allow EVEX2VEXOverride to explicitly specify a match. const CodeGenInstruction *VEXInst = nullptr; if (!EVEXInst->TheDef->isValueUnset("EVEX2VEXOverride")) { StringRef AltInstStr = - EVEXInst->TheDef->getValueAsString("EVEX2VEXOverride"); + EVEXInst->TheDef->getValueAsString("EVEX2VEXOverride"); Record *AltInstRec = Records.getDef(AltInstStr); assert(AltInstRec && "EVEX2VEXOverride instruction not found!"); VEXInst = &Target.getInstruction(AltInstRec); } else { - auto Match = llvm::find_if(VEXInsts[Opcode], IsMatch(EVEXInst)); - if (Match != VEXInsts[Opcode].end()) + auto Match = llvm::find_if(CompressedInsts[Opcode], IsMatch(EVEXInst)); + if (Match != CompressedInsts[Opcode].end()) VEXInst = *Match; } @@ -206,5 +205,5 @@ void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { } } // namespace -static TableGen::Emitter::OptClass - X("gen-x86-EVEX2VEX-tables", "Generate X86 EVEX to VEX compress tables"); +static TableGen::Emitter::OptClass + X("gen-x86-compress-evex-tables", "Generate X86 EVEX compression tables"); -- GitLab From 241e4c7466b877265e1645ca4709fe666c95c6c4 Mon Sep 17 00:00:00 2001 From: NAKAMURA Takumi Date: Sat, 6 Jan 2024 13:39:22 +0900 Subject: [PATCH 185/728] [Bazel] Fixup for #77008 (`orc::SymbolMap`) --- utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index f035a17833d8..6d162300169c 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -2719,6 +2719,7 @@ cc_library( ]), hdrs = glob([ "include/llvm/ExecutionEngine/JITLink/*.h", + "include/llvm/ExecutionEngine/Orc/*.h", ]), copts = llvm_copts, deps = [ -- GitLab From 16094cb629159ee0896e2ca1facc15118b229665 Mon Sep 17 00:00:00 2001 From: hev Date: Sat, 6 Jan 2024 13:36:09 +0800 Subject: [PATCH 186/728] [llvm][LoongArch] Support per-global code model attribute for LoongArch (#72079) This patch gets the code model from global variable attribute if it has, otherwise the target's will be used. --------- Signed-off-by: WANG Rui --- .../LoongArch/LoongArchISelLowering.cpp | 22 +++++++--- .../Target/LoongArch/LoongArchISelLowering.h | 3 +- .../LoongArch/global-variable-code-model.ll | 44 +++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 llvm/test/CodeGen/LoongArch/global-variable-code-model.ll diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 2cfb2c1d7811..3e75b9fa5230 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -762,12 +762,13 @@ static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty, template SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG, + CodeModel::Model M, bool IsLocal) const { SDLoc DL(N); EVT Ty = getPointerTy(DAG.getDataLayout()); SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0); - switch (DAG.getTarget().getCodeModel()) { + switch (M) { default: report_fatal_error("Unsupported code model"); @@ -808,24 +809,35 @@ SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG, SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { - return getAddr(cast(Op), DAG); + return getAddr(cast(Op), DAG, + DAG.getTarget().getCodeModel()); } SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op, SelectionDAG &DAG) const { - return getAddr(cast(Op), DAG); + return getAddr(cast(Op), DAG, + DAG.getTarget().getCodeModel()); } SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op, SelectionDAG &DAG) const { - return getAddr(cast(Op), DAG); + return getAddr(cast(Op), DAG, + DAG.getTarget().getCodeModel()); } SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { GlobalAddressSDNode *N = cast(Op); assert(N->getOffset() == 0 && "unexpected offset in global node"); - return getAddr(N, DAG, N->getGlobal()->isDSOLocal()); + auto CM = DAG.getTarget().getCodeModel(); + const GlobalValue *GV = N->getGlobal(); + + if (GV->isDSOLocal() && isa(GV)) { + if (auto GCM = dyn_cast(GV)->getCodeModel()) + CM = *GCM; + } + + return getAddr(N, DAG, CM, GV->isDSOLocal()); } SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N, diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index 2875aa82e424..72182623b2c3 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -254,7 +254,8 @@ private: LoongArchCCAssignFn Fn) const; template - SDValue getAddr(NodeTy *N, SelectionDAG &DAG, bool IsLocal = true) const; + SDValue getAddr(NodeTy *N, SelectionDAG &DAG, CodeModel::Model M, + bool IsLocal = true) const; SDValue getStaticTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG, unsigned Opc, bool Large = false) const; SDValue getDynamicTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG, diff --git a/llvm/test/CodeGen/LoongArch/global-variable-code-model.ll b/llvm/test/CodeGen/LoongArch/global-variable-code-model.ll new file mode 100644 index 000000000000..aa4780834ac3 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/global-variable-code-model.ll @@ -0,0 +1,44 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s + +@a= external dso_local global i32, code_model "small", align 4 + +define dso_local signext i32 @local_small() #0 { +; CHECK-LABEL: local_small: +; CHECK: # %bb.0: +; CHECK-NEXT: pcalau12i $a0, %pc_hi20(a) +; CHECK-NEXT: addi.d $a0, $a0, %pc_lo12(a) +; CHECK-NEXT: ld.w $a0, $a0, 0 +; CHECK-NEXT: ret + %1 = load i32, ptr @a, align 4 + ret i32 %1 +} + +@b= external dso_local global i32, code_model "large", align 4 + +define dso_local signext i32 @local_large() #0 { +; CHECK-LABEL: local_large: +; CHECK: # %bb.0: +; CHECK-NEXT: pcalau12i $a0, %pc_hi20(b) +; CHECK-NEXT: addi.d $t8, $zero, %pc_lo12(b) +; CHECK-NEXT: lu32i.d $t8, %pc64_lo20(b) +; CHECK-NEXT: lu52i.d $t8, $t8, %pc64_hi12(b) +; CHECK-NEXT: add.d $a0, $t8, $a0 +; CHECK-NEXT: ld.w $a0, $a0, 0 +; CHECK-NEXT: ret + %1 = load i32, ptr @b, align 4 + ret i32 %1 +} + +@c= external global i32, code_model "large", align 4 + +define dso_local signext i32 @non_local_large() #0 { +; CHECK-LABEL: non_local_large: +; CHECK: # %bb.0: +; CHECK-NEXT: pcalau12i $a0, %got_pc_hi20(c) +; CHECK-NEXT: ld.d $a0, $a0, %got_pc_lo12(c) +; CHECK-NEXT: ld.w $a0, $a0, 0 +; CHECK-NEXT: ret + %1 = load i32, ptr @c, align 4 + ret i32 %1 +} -- GitLab From 602c8fa2d8da6562e4f36df3bd63c26a4c7461e7 Mon Sep 17 00:00:00 2001 From: Craig Hesling Date: Sat, 6 Jan 2024 01:08:59 -0500 Subject: [PATCH 187/728] [GitHub] Fix slow sccache install on macOS by upgrading macOS version (#77165) The "Setup ccache" step on macOS-11 builds takes between 15 to 20 mins, whereas this step takes a less than 10 seconds on other runners. The bulk of this time is spent at the "Install sccache" step, where brew emits warnings like "Warning: You are using macOS 11." and "We (and Apple) do not provide support for this old version...". Bumping the version of macOS greatly decreases this cache setup time to about 20 seconds. Furthermore, it seems like it is speeding up general build times, too. It appears that https://github.com/actions/virtual-environments/issues/5900 has been resolved or obsoleted, so I do not believe we need to lock macOS to 11 anymore. --- .github/workflows/llvm-project-tests.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml index 02b1ab75e960..3345e734e1eb 100644 --- a/.github/workflows/llvm-project-tests.yml +++ b/.github/workflows/llvm-project-tests.yml @@ -14,7 +14,7 @@ on: required: false os_list: required: false - default: '["ubuntu-latest", "windows-2019", "macOS-11"]' + default: '["ubuntu-latest", "windows-2019", "macOS-12"]' workflow_call: inputs: build_target: @@ -34,9 +34,7 @@ on: type: string # Use windows-2019 due to: # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317 - # We're using a specific version of macOS due to: - # https://github.com/actions/virtual-environments/issues/5900 - default: '["ubuntu-latest", "windows-2019", "macOS-11"]' + default: '["ubuntu-latest", "windows-2019", "macOS-12"]' concurrency: # Skip intermediate builds: always. @@ -91,10 +89,6 @@ jobs: variant: sccache - name: Build and Test uses: llvm/actions/build-test-llvm-project@main - env: - # Workaround for https://github.com/actions/virtual-environments/issues/5900. - # This should be a no-op for non-mac OSes - PKG_CONFIG_PATH: /usr/local/Homebrew/Library/Homebrew/os/mac/pkgconfig//12 with: cmake_args: '-GNinja -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLDB_INCLUDE_TESTS=OFF -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ${{ inputs.extra_cmake_args }}' build_target: '${{ inputs.build_target }}' -- GitLab From cf02e6e71064ba2ce36c354e3bd6b2d57de29d85 Mon Sep 17 00:00:00 2001 From: Craig Hesling Date: Sat, 6 Jan 2024 01:15:10 -0500 Subject: [PATCH 188/728] [GitHub] Remove redundant cache key prefix (#76914) Remove the redundant sccache cache key prefix. This prefix is already added by the ccache action, which results in cache keys like "sccache-sccache-ubuntu-...". See the following source lines as proof: https://github.com/hendrikmuhs/ccache-action/blob/2a51777f6f64b7b7bea213601acba8f5f4fdbe03/src/restore.ts#L22-L23 --- .github/workflows/llvm-project-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml index 3345e734e1eb..594831ee6b5f 100644 --- a/.github/workflows/llvm-project-tests.yml +++ b/.github/workflows/llvm-project-tests.yml @@ -85,7 +85,7 @@ jobs: # enough cache space for all the tests to run at once and still # fit under the 10 GB limit. max-size: 500M - key: sccache-${{ matrix.os }} + key: ${{ matrix.os }} variant: sccache - name: Build and Test uses: llvm/actions/build-test-llvm-project@main -- GitLab From 376baeb2d535826eb2d8158c4147e37cda493f35 Mon Sep 17 00:00:00 2001 From: Craig Hesling Date: Sat, 6 Jan 2024 01:22:07 -0500 Subject: [PATCH 189/728] [GitHub] Add basic CI for libclang Python binding unit tests (#76784) This is important to aid development of Python type annotations in the libclang binding. See https://github.com/llvm/llvm-project/issues/76664 for more details. * Run on all pull requests and direct pushes. * This makes use of the existing llvm-project-tests.yml recipe, which will preload ccache from previous runs. * Building libclang currently takes about 9mins when ccache is warm and about an 1hr 20mins if it is cold using the standard GitHub ubuntu runner. * In the future, this could be broken into the following discrete steps for clarity: 1. Build libclang dependency. ninja -C build libclang 2. Run Python unit tests. ninja -C build check-clang-python * Followup changes will bring testing on older python versions and static type checking. Issue https://github.com/llvm/llvm-project/issues/76601. --- .github/workflows/libclang-python-tests.yml | 39 +++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/libclang-python-tests.yml diff --git a/.github/workflows/libclang-python-tests.yml b/.github/workflows/libclang-python-tests.yml new file mode 100644 index 000000000000..73edb6cf3bad --- /dev/null +++ b/.github/workflows/libclang-python-tests.yml @@ -0,0 +1,39 @@ +name: Libclang Python Binding Tests + +permissions: + contents: read + +on: + workflow_dispatch: + push: + paths: + - 'clang/bindings/python/**' + - 'clang/tools/libclang/**' + - 'clang/CMakeList.txt' + - '.github/workflows/libclang-python-tests.yml' + - '.github/workflows/llvm-project-tests.yml' + pull_request: + paths: + - 'clang/bindings/python/**' + - 'clang/tools/libclang/**' + - 'clang/CMakeList.txt' + - '.github/workflows/libclang-python-tests.yml' + - '.github/workflows/llvm-project-tests.yml' + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + check-clang-python: + # Build libclang and then run the libclang Python binding's unit tests. + name: Build and run Python unit tests + uses: ./.github/workflows/llvm-project-tests.yml + with: + build_target: check-clang-python + projects: clang + # There is an issue running on "windows-2019". + # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082. + os_list: '["ubuntu-latest"]' -- GitLab From 80dbf601d1815ff90b5aee18f426da964920dbe7 Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Sat, 6 Jan 2024 15:29:25 +0800 Subject: [PATCH 190/728] [X86][NFC] Remove EVEX2VEXOverride/NotEVEX2VEXConvertible Remove these two classes and put all the entries in X86 EVEX compression tables that need special handling in .def file. PR #77065 tries to add entries that need special handling for APX in .def file. Compared to setting fields in td files, that method looks cleaner. This patch is to unify the addition of manual entries. --- llvm/lib/Target/X86/X86InstrAVX512.td | 230 +++++++----------- llvm/lib/Target/X86/X86InstrFormats.td | 4 - llvm/lib/Target/X86/X86InstrUtils.td | 4 - .../TableGen/X86CompressEVEXTablesEmitter.cpp | 53 ++-- .../TableGen/X86ManualCompressEVEXTables.def | 88 +++++++ 5 files changed, 210 insertions(+), 169 deletions(-) create mode 100644 llvm/utils/TableGen/X86ManualCompressEVEXTables.def diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index c3a673f97d34..66646714ca77 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -3185,15 +3185,13 @@ defm : operation_subvector_mask_lowering; multiclass avx512_load opc, string OpcodeStr, string Name, X86VectorVTInfo _, PatFrag ld_frag, PatFrag mload, - X86SchedWriteMoveLS Sched, string EVEX2VEXOvrd, - bit NoRMPattern = 0, + X86SchedWriteMoveLS Sched, bit NoRMPattern = 0, SDPatternOperator SelectOprr = vselect> { let hasSideEffects = 0 in { let isMoveReg = 1 in def rr : AVX512PI, EVEX, Sched<[Sched.RR]>, - EVEX2VEXOverride; + _.ExeDomain>, EVEX, Sched<[Sched.RR]>; def rrkz : AVX512PI opc, string OpcodeStr, string Name, !if(NoRMPattern, [], [(set _.RC:$dst, (_.VT (ld_frag addr:$src)))]), - _.ExeDomain>, EVEX, Sched<[Sched.RM]>, - EVEX2VEXOverride; + _.ExeDomain>, EVEX, Sched<[Sched.RM]>; let Constraints = "$src0 = $dst", isConvertibleToThreeAddress = 1 in { def rrk : AVX512PI opc, string OpcodeStr, string Name, multiclass avx512_alignedload_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd, X86SchedWriteMoveLSWidths Sched, - string EVEX2VEXOvrd, bit NoRMPattern = 0> { + bit NoRMPattern = 0> { let Predicates = [prd] in defm Z : avx512_load, EVEX_V512; + Sched.ZMM, NoRMPattern>, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_load, EVEX_V256; + Sched.YMM, NoRMPattern>, EVEX_V256; defm Z128 : avx512_load, EVEX_V128; + Sched.XMM, NoRMPattern>, EVEX_V128; } } multiclass avx512_load_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd, X86SchedWriteMoveLSWidths Sched, - string EVEX2VEXOvrd, bit NoRMPattern = 0, + bit NoRMPattern = 0, SDPatternOperator SelectOprr = vselect> { let Predicates = [prd] in defm Z : avx512_load, EVEX_V512; + masked_load, Sched.ZMM, NoRMPattern, SelectOprr>, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_load, EVEX_V256; + masked_load, Sched.YMM, NoRMPattern, SelectOprr>, EVEX_V256; defm Z128 : avx512_load, EVEX_V128; + masked_load, Sched.XMM, NoRMPattern, SelectOprr>, EVEX_V128; } } multiclass avx512_store opc, string OpcodeStr, string BaseName, X86VectorVTInfo _, PatFrag st_frag, PatFrag mstore, - X86SchedWriteMoveLS Sched, string EVEX2VEXOvrd, - bit NoMRPattern = 0> { + X86SchedWriteMoveLS Sched, bit NoMRPattern = 0> { let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in { let isMoveReg = 1 in def rr_REV : AVX512PI, EVEX, - Sched<[Sched.RR]>, - EVEX2VEXOverride; + Sched<[Sched.RR]>; def rrk_REV : AVX512PI opc, string OpcodeStr, string BaseName, !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), !if(NoMRPattern, [], [(st_frag (_.VT _.RC:$src), addr:$dst)]), - _.ExeDomain>, EVEX, Sched<[Sched.MR]>, - EVEX2VEXOverride; + _.ExeDomain>, EVEX, Sched<[Sched.MR]>; def mrk : AVX512PI opc, string OpcodeStr, string BaseName, multiclass avx512_store_vl< bits<8> opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd, X86SchedWriteMoveLSWidths Sched, - string EVEX2VEXOvrd, bit NoMRPattern = 0> { + bit NoMRPattern = 0> { let Predicates = [prd] in defm Z : avx512_store, EVEX_V512; + masked_store, Sched.ZMM, NoMRPattern>, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_store, EVEX_V256; + masked_store, Sched.YMM, NoMRPattern>, EVEX_V256; defm Z128 : avx512_store, EVEX_V128; + masked_store, Sched.XMM, NoMRPattern>, EVEX_V128; } } multiclass avx512_alignedstore_vl opc, string OpcodeStr, AVX512VLVectorVTInfo _, Predicate prd, X86SchedWriteMoveLSWidths Sched, - string EVEX2VEXOvrd, bit NoMRPattern = 0> { + bit NoMRPattern = 0> { let Predicates = [prd] in defm Z : avx512_store, EVEX_V512; + masked_store_aligned, Sched.ZMM, NoMRPattern>, EVEX_V512; let Predicates = [prd, HasVLX] in { defm Z256 : avx512_store, EVEX_V256; + masked_store_aligned, Sched.YMM, NoMRPattern>, EVEX_V256; defm Z128 : avx512_store, EVEX_V128; + masked_store_aligned, Sched.XMM, NoMRPattern>, EVEX_V128; } } defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info, - HasAVX512, SchedWriteFMoveLS, "VMOVAPS">, + HasAVX512, SchedWriteFMoveLS>, avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info, - HasAVX512, SchedWriteFMoveLS, "VMOVAPS">, + HasAVX512, SchedWriteFMoveLS>, TB, EVEX_CD8<32, CD8VF>; defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info, - HasAVX512, SchedWriteFMoveLS, "VMOVAPD">, + HasAVX512, SchedWriteFMoveLS>, avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info, - HasAVX512, SchedWriteFMoveLS, "VMOVAPD">, + HasAVX512, SchedWriteFMoveLS>, TB, PD, REX_W, EVEX_CD8<64, CD8VF>; defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512, - SchedWriteFMoveLS, "VMOVUPS", 0, null_frag>, + SchedWriteFMoveLS, 0, null_frag>, avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512, - SchedWriteFMoveLS, "VMOVUPS">, + SchedWriteFMoveLS>, TB, EVEX_CD8<32, CD8VF>; defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512, - SchedWriteFMoveLS, "VMOVUPD", 0, null_frag>, + SchedWriteFMoveLS, 0, null_frag>, avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512, - SchedWriteFMoveLS, "VMOVUPD">, + SchedWriteFMoveLS>, TB, PD, REX_W, EVEX_CD8<64, CD8VF>; defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info, - HasAVX512, SchedWriteVecMoveLS, - "VMOVDQA", 1>, + HasAVX512, SchedWriteVecMoveLS, 1>, avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info, - HasAVX512, SchedWriteVecMoveLS, - "VMOVDQA", 1>, + HasAVX512, SchedWriteVecMoveLS, 1>, TB, PD, EVEX_CD8<32, CD8VF>; defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info, - HasAVX512, SchedWriteVecMoveLS, - "VMOVDQA">, + HasAVX512, SchedWriteVecMoveLS>, avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info, - HasAVX512, SchedWriteVecMoveLS, - "VMOVDQA">, + HasAVX512, SchedWriteVecMoveLS>, TB, PD, REX_W, EVEX_CD8<64, CD8VF>; defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI, - SchedWriteVecMoveLS, "VMOVDQU", 1>, + SchedWriteVecMoveLS, 1>, avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info, HasBWI, - SchedWriteVecMoveLS, "VMOVDQU", 1>, + SchedWriteVecMoveLS, 1>, TB, XD, EVEX_CD8<8, CD8VF>; defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI, - SchedWriteVecMoveLS, "VMOVDQU", 1>, + SchedWriteVecMoveLS, 1>, avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info, HasBWI, - SchedWriteVecMoveLS, "VMOVDQU", 1>, + SchedWriteVecMoveLS, 1>, TB, XD, REX_W, EVEX_CD8<16, CD8VF>; defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512, - SchedWriteVecMoveLS, "VMOVDQU", 1, null_frag>, + SchedWriteVecMoveLS, 1, null_frag>, avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info, HasAVX512, - SchedWriteVecMoveLS, "VMOVDQU", 1>, + SchedWriteVecMoveLS, 1>, TB, XS, EVEX_CD8<32, CD8VF>; defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512, - SchedWriteVecMoveLS, "VMOVDQU", 0, null_frag>, + SchedWriteVecMoveLS, 0, null_frag>, avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info, HasAVX512, - SchedWriteVecMoveLS, "VMOVDQU">, + SchedWriteVecMoveLS>, TB, XS, REX_W, EVEX_CD8<64, CD8VF>; // Special instructions to help with spilling when we don't have VLX. We need @@ -4844,8 +4825,7 @@ defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul, defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul, SchedWriteVecIMul, HasBWI, 1>; defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul, - SchedWriteVecIMul, HasDQI, 1>, T8, - NotEVEX2VEXConvertible; + SchedWriteVecIMul, HasDQI, 1>, T8; defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SchedWriteVecIMul, HasBWI, 1>; defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SchedWriteVecIMul, @@ -4989,8 +4969,7 @@ defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax, defm VPMAXSD : avx512_binop_rm_vl_d<0x3D, "vpmaxsd", smax, SchedWriteVecALU, HasAVX512, 1>, T8; defm VPMAXSQ : avx512_binop_rm_vl_q<0x3D, "vpmaxsq", smax, - SchedWriteVecALU, HasAVX512, 1>, T8, - NotEVEX2VEXConvertible; + SchedWriteVecALU, HasAVX512, 1>, T8; defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax, SchedWriteVecALU, HasBWI, 1>; @@ -4999,8 +4978,7 @@ defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax, defm VPMAXUD : avx512_binop_rm_vl_d<0x3F, "vpmaxud", umax, SchedWriteVecALU, HasAVX512, 1>, T8; defm VPMAXUQ : avx512_binop_rm_vl_q<0x3F, "vpmaxuq", umax, - SchedWriteVecALU, HasAVX512, 1>, T8, - NotEVEX2VEXConvertible; + SchedWriteVecALU, HasAVX512, 1>, T8; defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin, SchedWriteVecALU, HasBWI, 1>, T8; @@ -5009,8 +4987,7 @@ defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin, defm VPMINSD : avx512_binop_rm_vl_d<0x39, "vpminsd", smin, SchedWriteVecALU, HasAVX512, 1>, T8; defm VPMINSQ : avx512_binop_rm_vl_q<0x39, "vpminsq", smin, - SchedWriteVecALU, HasAVX512, 1>, T8, - NotEVEX2VEXConvertible; + SchedWriteVecALU, HasAVX512, 1>, T8; defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin, SchedWriteVecALU, HasBWI, 1>; @@ -5019,8 +4996,7 @@ defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin, defm VPMINUD : avx512_binop_rm_vl_d<0x3B, "vpminud", umin, SchedWriteVecALU, HasAVX512, 1>, T8; defm VPMINUQ : avx512_binop_rm_vl_q<0x3B, "vpminuq", umin, - SchedWriteVecALU, HasAVX512, 1>, T8, - NotEVEX2VEXConvertible; + SchedWriteVecALU, HasAVX512, 1>, T8; // PMULLQ: Use 512bit version to implement 128/256 bit in case NoVLX. let Predicates = [HasDQI, NoVLX] in { @@ -5405,8 +5381,7 @@ multiclass avx512_fp_scalar_round opc, string OpcodeStr,X86VectorVTInfo } multiclass avx512_fp_scalar_sae opc, string OpcodeStr,X86VectorVTInfo _, SDNode OpNode, SDNode VecNode, SDNode SaeNode, - X86FoldableSchedWrite sched, bit IsCommutable, - string EVEX2VexOvrd> { + X86FoldableSchedWrite sched, bit IsCommutable> { let ExeDomain = _.ExeDomain in { defm rr_Int : AVX512_maskable_scalar opc, string OpcodeStr,X86VectorVTInfo _, (ins _.FRC:$src1, _.FRC:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>, - Sched<[sched]>, - EVEX2VEXOverride { + Sched<[sched]> { let isCommutable = IsCommutable; } def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst), @@ -5436,8 +5410,7 @@ multiclass avx512_fp_scalar_sae opc, string OpcodeStr,X86VectorVTInfo _, OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set _.FRC:$dst, (OpNode _.FRC:$src1, (_.ScalarLdFrag addr:$src2)))]>, - Sched<[sched.Folded, sched.ReadAfterFold]>, - EVEX2VEXOverride; + Sched<[sched.Folded, sched.ReadAfterFold]>; } let Uses = [MXCSR] in @@ -5474,19 +5447,15 @@ multiclass avx512_binop_s_sae opc, string OpcodeStr, SDNode OpNode, SDNode VecNode, SDNode SaeNode, X86SchedWriteSizes sched, bit IsCommutable> { defm SSZ : avx512_fp_scalar_sae, + VecNode, SaeNode, sched.PS.Scl, IsCommutable>, TB, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>; defm SDZ : avx512_fp_scalar_sae, + VecNode, SaeNode, sched.PD.Scl, IsCommutable>, TB, XD, REX_W, EVEX, VVVV, VEX_LIG, EVEX_CD8<64, CD8VT1>; let Predicates = [HasFP16] in { defm SHZ : avx512_fp_scalar_sae, - T_MAP5, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>, - NotEVEX2VEXConvertible; + VecNode, SaeNode, sched.PH.Scl, IsCommutable>, + T_MAP5, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>; } } defm VADD : avx512_binop_s_round<0x58, "vadd", any_fadd, X86fadds, X86faddRnds, @@ -5506,14 +5475,13 @@ defm VMAX : avx512_binop_s_sae<0x5F, "vmax", X86fmax, X86fmaxs, X86fmaxSAEs, // X86fminc and X86fmaxc instead of X86fmin and X86fmax multiclass avx512_comutable_binop_s opc, string OpcodeStr, X86VectorVTInfo _, SDNode OpNode, - X86FoldableSchedWrite sched, - string EVEX2VEXOvrd> { + X86FoldableSchedWrite sched> { let isCodeGenOnly = 1, Predicates = [HasAVX512], ExeDomain = _.ExeDomain in { def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst), (ins _.FRC:$src1, _.FRC:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>, - Sched<[sched]>, EVEX2VEXOverride { + Sched<[sched]> { let isCommutable = 1; } def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst), @@ -5521,36 +5489,34 @@ multiclass avx512_comutable_binop_s opc, string OpcodeStr, OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set _.FRC:$dst, (OpNode _.FRC:$src1, (_.ScalarLdFrag addr:$src2)))]>, - Sched<[sched.Folded, sched.ReadAfterFold]>, - EVEX2VEXOverride; + Sched<[sched.Folded, sched.ReadAfterFold]>; } } defm VMINCSSZ : avx512_comutable_binop_s<0x5D, "vminss", f32x_info, X86fminc, - SchedWriteFCmp.Scl, "VMINCSS">, TB, XS, + SchedWriteFCmp.Scl>, TB, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>, SIMD_EXC; defm VMINCSDZ : avx512_comutable_binop_s<0x5D, "vminsd", f64x_info, X86fminc, - SchedWriteFCmp.Scl, "VMINCSD">, TB, XD, + SchedWriteFCmp.Scl>, TB, XD, REX_W, EVEX, VVVV, VEX_LIG, EVEX_CD8<64, CD8VT1>, SIMD_EXC; defm VMAXCSSZ : avx512_comutable_binop_s<0x5F, "vmaxss", f32x_info, X86fmaxc, - SchedWriteFCmp.Scl, "VMAXCSS">, TB, XS, + SchedWriteFCmp.Scl>, TB, XS, EVEX, VVVV, VEX_LIG, EVEX_CD8<32, CD8VT1>, SIMD_EXC; defm VMAXCSDZ : avx512_comutable_binop_s<0x5F, "vmaxsd", f64x_info, X86fmaxc, - SchedWriteFCmp.Scl, "VMAXCSD">, TB, XD, + SchedWriteFCmp.Scl>, TB, XD, REX_W, EVEX, VVVV, VEX_LIG, EVEX_CD8<64, CD8VT1>, SIMD_EXC; defm VMINCSHZ : avx512_comutable_binop_s<0x5D, "vminsh", f16x_info, X86fminc, - SchedWriteFCmp.Scl, "VMINCSH">, T_MAP5, XS, - EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>, SIMD_EXC, - NotEVEX2VEXConvertible; + SchedWriteFCmp.Scl>, T_MAP5, XS, + EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>, SIMD_EXC; + defm VMAXCSHZ : avx512_comutable_binop_s<0x5F, "vmaxsh", f16x_info, X86fmaxc, - SchedWriteFCmp.Scl, "VMAXCSH">, T_MAP5, XS, - EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>, SIMD_EXC, - NotEVEX2VEXConvertible; + SchedWriteFCmp.Scl>, T_MAP5, XS, + EVEX, VVVV, VEX_LIG, EVEX_CD8<16, CD8VT1>, SIMD_EXC; multiclass avx512_fp_packed opc, string OpcodeStr, SDPatternOperator OpNode, SDPatternOperator MaskOpNode, @@ -5820,8 +5786,7 @@ multiclass avx512_fp_scalef_all opc, bits<8> opcScaler, string OpcodeStr EVEX_V256, EVEX_CD8<16, CD8VF>, T_MAP6, PD; } } -defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", - SchedWriteFAdd>, NotEVEX2VEXConvertible; +defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", SchedWriteFAdd>; //===----------------------------------------------------------------------===// // AVX-512 VPTESTM instructions @@ -5985,11 +5950,9 @@ multiclass avx512_shift_sizes opc, string OpcodeStr, SDNode OpNode, multiclass avx512_shift_types opcd, bits<8> opcq, bits<8> opcw, string OpcodeStr, SDNode OpNode, - X86SchedWriteWidths sched, - bit NotEVEX2VEXConvertibleQ = 0> { + X86SchedWriteWidths sched> { defm D : avx512_shift_sizes; - let notEVEX2VEXConvertible = NotEVEX2VEXConvertibleQ in defm Q : avx512_shift_sizes, REX_W; defm W : avx512_shift_sizes opcw, Format ImmFormR, Format ImmFormM, multiclass avx512_shift_rmi_dq opcd, bits<8> opcq, Format ImmFormR, Format ImmFormM, string OpcodeStr, SDNode OpNode, - X86SchedWriteWidths sched, - bit NotEVEX2VEXConvertibleQ = 0> { + X86SchedWriteWidths sched> { defm D: avx512_shift_rmi_sizes, EVEX_CD8<32, CD8VF>; - let notEVEX2VEXConvertible = NotEVEX2VEXConvertibleQ in defm Q: avx512_shift_rmi_sizes, EVEX_CD8<64, CD8VF>, REX_W; } @@ -6054,7 +6015,7 @@ defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli, SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX, VVVV; defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai, - SchedWriteVecShiftImm, 1>, + SchedWriteVecShiftImm>, avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai, SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX, VVVV; @@ -6066,7 +6027,7 @@ defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", X86vrotli, defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl, SchedWriteVecShift>; defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra, - SchedWriteVecShift, 1>; + SchedWriteVecShift>; defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl, SchedWriteVecShift>; @@ -8443,9 +8404,9 @@ multiclass avx512_cvtqq2pd opc, string OpcodeStr, SDPatternOperator OpNo } let Predicates = [HasDQI, HasVLX] in { defm Z128 : avx512_vcvt_fp, EVEX_V128, NotEVEX2VEXConvertible; + MaskOpNode, sched.XMM>, EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256, NotEVEX2VEXConvertible; + MaskOpNode, sched.YMM>, EVEX_V256; } } @@ -8524,11 +8485,10 @@ multiclass avx512_cvtqq2ps_dq2ph opc, string OpcodeStr, SDPatternOperato defm Z128 : avx512_vcvt_fp, - EVEX_V128, NotEVEX2VEXConvertible; + EVEX_V128; defm Z256 : avx512_vcvt_fp, EVEX_V256, - NotEVEX2VEXConvertible; + "{y}">, EVEX_V256; // Special patterns to allow use of X86VM[SU]intToFP for masking. Instruction // patterns have been disabled with null_frag. @@ -10882,8 +10842,7 @@ defm VGETMANTSH: avx512_common_fp_sae_scalar_imm<"vgetmantsh", f16x_info, multiclass avx512_shuff_packed_128_common opc, string OpcodeStr, X86FoldableSchedWrite sched, X86VectorVTInfo _, - X86VectorVTInfo CastInfo, - string EVEX2VEXOvrd> { + X86VectorVTInfo CastInfo> { let ExeDomain = _.ExeDomain in { defm rri : AVX512_maskable opc, string OpcodeStr, (_.VT (bitconvert (CastInfo.VT (X86Shuf128 _.RC:$src1, _.RC:$src2, (i8 timm:$src3)))))>, - Sched<[sched]>, EVEX2VEXOverride; + Sched<[sched]>; defm rmi : AVX512_maskable opc, string OpcodeStr, (CastInfo.VT (X86Shuf128 _.RC:$src1, (CastInfo.LdFrag addr:$src2), (i8 timm:$src3)))))>, - Sched<[sched.Folded, sched.ReadAfterFold]>, - EVEX2VEXOverride; + Sched<[sched.Folded, sched.ReadAfterFold]>; defm rmbi : AVX512_maskable opc, string OpcodeStr, multiclass avx512_shuff_packed_128 opc, - string EVEX2VEXOvrd>{ + AVX512VLVectorVTInfo CastInfo, bits<8> opc>{ let Predicates = [HasAVX512] in defm Z : avx512_shuff_packed_128_common, EVEX_V512; + _.info512, CastInfo.info512>, EVEX_V512; let Predicates = [HasAVX512, HasVLX] in defm Z256 : avx512_shuff_packed_128_common, EVEX_V256; + _.info256, CastInfo.info256>, EVEX_V256; } defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4", WriteFShuffle256, - avx512vl_f32_info, avx512vl_f64_info, 0x23, "VPERM2F128">, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>; + avx512vl_f32_info, avx512vl_f64_info, 0x23>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>; defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2", WriteFShuffle256, - avx512vl_f64_info, avx512vl_f64_info, 0x23, "VPERM2F128">, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W; + avx512vl_f64_info, avx512vl_f64_info, 0x23>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W; defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4", WriteFShuffle256, - avx512vl_i32_info, avx512vl_i64_info, 0x43, "VPERM2I128">, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>; + avx512vl_i32_info, avx512vl_i64_info, 0x43>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<32, CD8VF>; defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2", WriteFShuffle256, - avx512vl_i64_info, avx512vl_i64_info, 0x43, "VPERM2I128">, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W; + avx512vl_i64_info, avx512vl_i64_info, 0x43>, AVX512AIi8Base, EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W; multiclass avx512_valign opc, string OpcodeStr, X86FoldableSchedWrite sched, X86VectorVTInfo _>{ - // NOTE: EVEX2VEXOverride changed back to Unset for 256-bit at the - // instantiation of this class. let ExeDomain = _.ExeDomain in { defm rri : AVX512_maskable, - Sched<[sched]>, EVEX2VEXOverride<"VPALIGNRrri">; + Sched<[sched]>; defm rmi : AVX512_maskable, - Sched<[sched.Folded, sched.ReadAfterFold]>, - EVEX2VEXOverride<"VPALIGNRrmi">; + Sched<[sched.Folded, sched.ReadAfterFold]>; defm rmbi : AVX512_maskable, AVX512AIi8Base, EVEX, VVVV, EVEX_V128; // We can't really override the 256-bit version so change it back to unset. - let EVEX2VEXOverride = ? in defm Z256 : avx512_valign<0x03, OpcodeStr, sched.YMM, _.info256>, AVX512AIi8Base, EVEX, VVVV, EVEX_V256; } @@ -11111,7 +11063,7 @@ let Predicates = [HasVLX, HasBWI] in { defm VDBPSADBW: avx512_common_3Op_rm_imm8<0x42, X86dbpsadbw, "vdbpsadbw", SchedWritePSADBW, avx512vl_i16_info, avx512vl_i8_info>, - EVEX_CD8<8, CD8VF>, NotEVEX2VEXConvertible; + EVEX_CD8<8, CD8VF>; multiclass avx512_unary_rm opc, string OpcodeStr, SDNode OpNode, X86FoldableSchedWrite sched, X86VectorVTInfo _> { @@ -13088,12 +13040,10 @@ multiclass avx512_cvtqq2ph opc, string OpcodeStr, SDPatternOperator OpNo let Predicates = [HasFP16, HasVLX] in { defm Z128 : avx512_vcvt_fp, - EVEX_V128, NotEVEX2VEXConvertible; + i128mem, VK2WM>, EVEX_V128; defm Z256 : avx512_vcvt_fp, - EVEX_V256, NotEVEX2VEXConvertible; + i256mem, VK4WM>, EVEX_V256; } def : InstAlias opcod, Format f, ImmType i, dag outs, dag ins, CD8_EltSize, !srl(VectSize, CD8_Form{1-0}))), 0); - // Used to prevent an explicit EVEX2VEX override for this instruction. - string EVEX2VEXOverride = ?; - - bit notEVEX2VEXConvertible = 0; // Prevent EVEX->VEX conversion. ExplicitOpPrefix explicitOpPrefix = NoExplicitOpPrefix; bits<2> explicitOpPrefixBits = explicitOpPrefix.Value; // TSFlags layout should be kept in sync with X86BaseInfo.h. diff --git a/llvm/lib/Target/X86/X86InstrUtils.td b/llvm/lib/Target/X86/X86InstrUtils.td index 132941a5734c..9183bcd7017f 100644 --- a/llvm/lib/Target/X86/X86InstrUtils.td +++ b/llvm/lib/Target/X86/X86InstrUtils.td @@ -66,9 +66,6 @@ class EVEX_CD8 { } class NoCD8 { bits<7> CD8_Scale = 0; } -class EVEX2VEXOverride { - string EVEX2VEXOverride = VEXInstrName; -} class AVX512BIi8Base : TB, PD { Domain ExeDomain = SSEPackedInt; ImmType ImmT = Imm8; @@ -89,7 +86,6 @@ class AVX512PDIi8Base : TB, PD { Domain ExeDomain = SSEPackedDouble; ImmType ImmT = Imm8; } -class NotEVEX2VEXConvertible { bit notEVEX2VEXConvertible = 1; } class ExplicitREX2Prefix { ExplicitOpPrefix explicitOpPrefix = ExplicitREX2; } class ExplicitVEXPrefix { ExplicitOpPrefix explicitOpPrefix = ExplicitVEX; } class ExplicitEVEXPrefix { ExplicitOpPrefix explicitOpPrefix = ExplicitEVEX; } diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index c1ea34dc02e2..3a26732cf323 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -17,12 +17,23 @@ #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" +#include +#include using namespace llvm; using namespace X86Disassembler; namespace { +const std::map ManualMap = { +#define ENTRY(OLD, NEW) {#OLD, #NEW}, +#include "X86ManualCompressEVEXTables.def" +}; +const std::set NoCompressSet = { +#define NOCOMP(INSN) #INSN, +#include "X86ManualCompressEVEXTables.def" +}; + class X86CompressEVEXTablesEmitter { RecordKeeper &Records; CodeGenTarget Target; @@ -151,13 +162,14 @@ void X86CompressEVEXTablesEmitter::run(raw_ostream &OS) { Target.getInstructionsByEnumValue(); for (const CodeGenInstruction *Inst : NumberedInstructions) { - const Record *Def = Inst->TheDef; - // Filter non-X86 instructions. - if (!Def->isSubClassOf("X86Inst")) - continue; + const Record *Rec = Inst->TheDef; // _REV instruction should not appear before encoding optimization - if (Def->getName().ends_with("_REV")) + if (!Rec->isSubClassOf("X86Inst") || Rec->getName().ends_with("_REV")) continue; + + if (NoCompressSet.find(Rec->getName()) != NoCompressSet.end()) + continue; + RecognizableInstrBase RI(*Inst); // Add VEX encoded instructions to one of CompressedInsts vectors according @@ -166,25 +178,24 @@ void X86CompressEVEXTablesEmitter::run(raw_ostream &OS) { CompressedInsts[RI.Opcode].push_back(Inst); // Add relevant EVEX encoded instructions to PreCompressionInsts else if (RI.Encoding == X86Local::EVEX && !RI.HasEVEX_K && !RI.HasEVEX_B && - !RI.HasEVEX_L2 && !Def->getValueAsBit("notEVEX2VEXConvertible")) + !RI.HasEVEX_L2) PreCompressionInsts.push_back(Inst); } - for (const CodeGenInstruction *EVEXInst : PreCompressionInsts) { + for (const CodeGenInstruction *Inst : PreCompressionInsts) { + const Record *Rec = Inst->TheDef; uint64_t Opcode = - getValueFromBitsInit(EVEXInst->TheDef->getValueAsBitsInit("Opcode")); - // For each EVEX instruction look for a VEX match in the appropriate vector - // (instructions with the same opcode) using function object IsMatch. - // Allow EVEX2VEXOverride to explicitly specify a match. + getValueFromBitsInit(Inst->TheDef->getValueAsBitsInit("Opcode")); const CodeGenInstruction *VEXInst = nullptr; - if (!EVEXInst->TheDef->isValueUnset("EVEX2VEXOverride")) { - StringRef AltInstStr = - EVEXInst->TheDef->getValueAsString("EVEX2VEXOverride"); - Record *AltInstRec = Records.getDef(AltInstStr); - assert(AltInstRec && "EVEX2VEXOverride instruction not found!"); - VEXInst = &Target.getInstruction(AltInstRec); + if (ManualMap.find(Rec->getName()) != ManualMap.end()) { + Record *NewRec = Records.getDef(ManualMap.at(Rec->getName())); + assert(NewRec && "Instruction not found!"); + VEXInst = &Target.getInstruction(NewRec); } else { - auto Match = llvm::find_if(CompressedInsts[Opcode], IsMatch(EVEXInst)); + // For each EVEX instruction look for a VEX match in the appropriate + // vector (instructions with the same opcode) using function object + // IsMatch. + auto Match = llvm::find_if(CompressedInsts[Opcode], IsMatch(Inst)); if (Match != CompressedInsts[Opcode].end()) VEXInst = *Match; } @@ -193,10 +204,10 @@ void X86CompressEVEXTablesEmitter::run(raw_ostream &OS) { continue; // In case a match is found add new entry to the appropriate table - if (EVEXInst->TheDef->getValueAsBit("hasVEX_L")) - EVEX2VEX256.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,1} + if (Rec->getValueAsBit("hasVEX_L")) + EVEX2VEX256.push_back(std::make_pair(Inst, VEXInst)); // {0,1} else - EVEX2VEX128.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,0} + EVEX2VEX128.push_back(std::make_pair(Inst, VEXInst)); // {0,0} } // Print both tables diff --git a/llvm/utils/TableGen/X86ManualCompressEVEXTables.def b/llvm/utils/TableGen/X86ManualCompressEVEXTables.def new file mode 100644 index 000000000000..0da32f92502c --- /dev/null +++ b/llvm/utils/TableGen/X86ManualCompressEVEXTables.def @@ -0,0 +1,88 @@ +//===- X86ManualCompressEVEXTables.def ---------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// \file +// This file defines all the entries in X86 EVEX compression tables that need +// special handling. +//===----------------------------------------------------------------------===// + +#ifndef NOCOMP +#define NOCOMP(INSN) +#endif +NOCOMP(VCVTQQ2PDZ128rr) +NOCOMP(VCVTQQ2PSZ128rm) +NOCOMP(VCVTQQ2PSZ128rr) +NOCOMP(VDBPSADBWZ128rmi) +NOCOMP(VDBPSADBWZ128rri) +NOCOMP(VPMAXSQZ128rm) +NOCOMP(VPMAXSQZ128rr) +NOCOMP(VPMAXUQZ128rm) +NOCOMP(VPMAXUQZ128rr) +NOCOMP(VPMINSQZ128rm) +NOCOMP(VPMINSQZ128rr) +NOCOMP(VPMINUQZ128rm) +NOCOMP(VPMINUQZ128rr) +NOCOMP(VPMULLQZ128rm) +NOCOMP(VPMULLQZ128rr) +NOCOMP(VPSRAQZ128ri) +NOCOMP(VPSRAQZ128rm) +NOCOMP(VPSRAQZ128rr) +NOCOMP(VSCALEFPSZ128rm) +NOCOMP(VDBPSADBWZ256rmi) +NOCOMP(VDBPSADBWZ256rri) +NOCOMP(VPMAXSQZ256rm) +NOCOMP(VPMAXSQZ256rr) +NOCOMP(VPMAXUQZ256rm) +NOCOMP(VPMAXUQZ256rr) +NOCOMP(VPMINSQZ256rm) +NOCOMP(VPMINSQZ256rr) +NOCOMP(VPMINUQZ256rm) +NOCOMP(VPMINUQZ256rr) +NOCOMP(VPMULLQZ256rm) +NOCOMP(VPMULLQZ256rr) +NOCOMP(VPSRAQZ256ri) +NOCOMP(VPSRAQZ256rm) +NOCOMP(VPSRAQZ256rr) +NOCOMP(VSCALEFPSZ256rm) +#undef NOCOMP + +#ifndef ENTRY +#define ENTRY(OLD, NEW) +#endif +ENTRY(VALIGNDZ128rmi, VPALIGNRrmi) +ENTRY(VALIGNDZ128rri, VPALIGNRrri) +ENTRY(VALIGNQZ128rmi, VPALIGNRrmi) +ENTRY(VALIGNQZ128rri, VPALIGNRrri) +ENTRY(VMAXSDZrm, VMAXSDrm) +ENTRY(VMAXSDZrr, VMAXSDrr) +ENTRY(VMAXSSZrm, VMAXSSrm) +ENTRY(VMAXSSZrr, VMAXSSrr) +ENTRY(VMINSDZrm, VMINSDrm) +ENTRY(VMINSDZrr, VMINSDrr) +ENTRY(VMINSSZrm, VMINSSrm) +ENTRY(VMINSSZrr, VMINSSrr) +ENTRY(VMOVDQU16Z128mr, VMOVDQUmr) +ENTRY(VMOVDQU16Z128rm, VMOVDQUrm) +ENTRY(VMOVDQU16Z128rr, VMOVDQUrr) +ENTRY(VMOVDQU8Z128mr, VMOVDQUmr) +ENTRY(VMOVDQU8Z128rm, VMOVDQUrm) +ENTRY(VMOVDQU8Z128rr, VMOVDQUrr) +ENTRY(VMOVDQU16Z256mr, VMOVDQUYmr) +ENTRY(VMOVDQU16Z256rm, VMOVDQUYrm) +ENTRY(VMOVDQU16Z256rr, VMOVDQUYrr) +ENTRY(VMOVDQU8Z256mr, VMOVDQUYmr) +ENTRY(VMOVDQU8Z256rm, VMOVDQUYrm) +ENTRY(VMOVDQU8Z256rr, VMOVDQUYrr) +ENTRY(VSHUFF32X4Z256rmi, VPERM2F128rm) +ENTRY(VSHUFF32X4Z256rri, VPERM2F128rr) +ENTRY(VSHUFF64X2Z256rmi, VPERM2F128rm) +ENTRY(VSHUFF64X2Z256rri, VPERM2F128rr) +ENTRY(VSHUFI32X4Z256rmi, VPERM2I128rm) +ENTRY(VSHUFI32X4Z256rri, VPERM2I128rr) +ENTRY(VSHUFI64X2Z256rmi, VPERM2I128rm) +ENTRY(VSHUFI64X2Z256rri, VPERM2I128rr) +#undef ENTRY -- GitLab From 04a7ec610ee1ad869e402c327984cb649be86f3c Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Sat, 6 Jan 2024 17:07:39 +0800 Subject: [PATCH 191/728] [X86][NFC] Remove VEX_W1X after 80dbf60 --- llvm/lib/Target/X86/X86InstrAVX512.td | 18 +- llvm/lib/Target/X86/X86InstrFormats.td | 2 - llvm/lib/Target/X86/X86InstrUtils.td | 2 - .../TableGen/X86CompressEVEXTablesEmitter.cpp | 45 ++-- llvm/utils/TableGen/X86FoldTablesEmitter.cpp | 6 +- .../TableGen/X86ManualCompressEVEXTables.def | 243 ++++++++++++++++++ 6 files changed, 271 insertions(+), 45 deletions(-) diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index 66646714ca77..fe7d90fbcdf7 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -448,7 +448,7 @@ multiclass vinsert_for_type, X86VectorVTInfo< 4, EltVT64, VR256X>, null_frag, vinsert128_insert, sched>, - VEX_W1X, EVEX_V256; + EVEX_V256, REX_W; // Even with DQI we'd like to only use these instructions for masking. let Predicates = [HasDQI] in { @@ -750,7 +750,7 @@ multiclass vextract_for_type, X86VectorVTInfo< 2, EltVT64, VR128X>, null_frag, vextract128_extract, SchedRR, SchedMR>, - VEX_W1X, EVEX_V256, EVEX_CD8<64, CD8VT2>; + EVEX_V256, EVEX_CD8<64, CD8VT2>, REX_W; // Even with DQI we'd like to only use these instructions for masking. let Predicates = [HasDQI] in { @@ -1161,7 +1161,7 @@ multiclass avx512_fp_broadcast_ss opc, string OpcodeStr, defm VBROADCASTSS : avx512_fp_broadcast_ss<0x18, "vbroadcastss", avx512vl_f32_info>; defm VBROADCASTSD : avx512_fp_broadcast_sd<0x19, "vbroadcastsd", - avx512vl_f64_info>, VEX_W1X; + avx512vl_f64_info>, REX_W; multiclass avx512_int_broadcast_reg opc, SchedWrite SchedRR, X86VectorVTInfo _, SDPatternOperator OpNode, @@ -1267,7 +1267,7 @@ defm VPBROADCASTW : avx512_int_broadcast_rm_vl<0x79, "vpbroadcastw", defm VPBROADCASTD : avx512_int_broadcast_rm_vl<0x58, "vpbroadcastd", avx512vl_i32_info, HasAVX512, 1>; defm VPBROADCASTQ : avx512_int_broadcast_rm_vl<0x59, "vpbroadcastq", - avx512vl_i64_info, HasAVX512, 1>, VEX_W1X; + avx512vl_i64_info, HasAVX512, 1>, REX_W; multiclass avx512_subvec_broadcast_rm opc, string OpcodeStr, SDPatternOperator OpNode, @@ -1460,11 +1460,11 @@ let Predicates = [HasBF16, HasVLX] in let Predicates = [HasVLX, HasDQI] in { defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm_dq<0x5a, "vbroadcasti64x2", - X86SubVBroadcastld128, v4i64x_info, v2i64x_info>, VEX_W1X, - EVEX_V256, EVEX_CD8<64, CD8VT2>; + X86SubVBroadcastld128, v4i64x_info, v2i64x_info>, + EVEX_V256, EVEX_CD8<64, CD8VT2>, REX_W; defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm_dq<0x1a, "vbroadcastf64x2", - X86SubVBroadcastld128, v4f64x_info, v2f64x_info>, VEX_W1X, - EVEX_V256, EVEX_CD8<64, CD8VT2>; + X86SubVBroadcastld128, v4f64x_info, v2f64x_info>, + EVEX_V256, EVEX_CD8<64, CD8VT2>, REX_W; // Patterns for selects of bitcasted operations. def : Pat<(vselect_mask VK4WM:$mask, @@ -6396,7 +6396,7 @@ defm VPERMILPS : avx512_permil<"vpermilps", 0x04, 0x0C, avx512vl_f32_info, avx512vl_i32_info>; let ExeDomain = SSEPackedDouble in defm VPERMILPD : avx512_permil<"vpermilpd", 0x05, 0x0D, avx512vl_f64_info, - avx512vl_i64_info>, VEX_W1X; + avx512vl_i64_info>, REX_W; //===----------------------------------------------------------------------===// // AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW diff --git a/llvm/lib/Target/X86/X86InstrFormats.td b/llvm/lib/Target/X86/X86InstrFormats.td index 9f0b73244534..8798b13a1761 100644 --- a/llvm/lib/Target/X86/X86InstrFormats.td +++ b/llvm/lib/Target/X86/X86InstrFormats.td @@ -247,8 +247,6 @@ class X86Inst opcod, Format f, ImmType i, dag outs, dag ins, bit hasREPPrefix = 0; // Does this inst have a REP prefix? bits<2> OpEncBits = OpEnc.Value; bit IgnoresW = 0; // Does this inst ignore REX_W field? - bit EVEX_W1_VEX_W0 = 0; // This EVEX inst with VEX.W==1 can become a VEX - // instruction with VEX.W == 0. bit hasVEX_4V = 0; // Does this inst require the VEX.VVVV field? bit hasVEX_L = 0; // Does this inst use large (256-bit) registers? bit ignoresVEX_L = 0; // Does this instruction ignore the L-bit diff --git a/llvm/lib/Target/X86/X86InstrUtils.td b/llvm/lib/Target/X86/X86InstrUtils.td index 9183bcd7017f..f4ae15837fbf 100644 --- a/llvm/lib/Target/X86/X86InstrUtils.td +++ b/llvm/lib/Target/X86/X86InstrUtils.td @@ -43,8 +43,6 @@ class XOP { Encoding OpEnc = EncXOP; } class VEX { Encoding OpEnc = EncVEX; } class EVEX { Encoding OpEnc = EncEVEX; } class WIG { bit IgnoresW = 1; } -// Special version of REX_W that can be changed to VEX.W==0 for EVEX2VEX. -class VEX_W1X { bit hasREX_W = 1; bit EVEX_W1_VEX_W0 = 1; } class VEX_L { bit hasVEX_L = 1; } class VEX_LIG { bit ignoresVEX_L = 1; } class VVVV { bit hasVEX_4V = 1; } diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index 3a26732cf323..a45e87af4a3f 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -95,34 +95,23 @@ static inline uint64_t getValueFromBitsInit(const BitsInit *B) { return Value; } -// Function object - Operator() returns true if the given VEX instruction -// matches the EVEX instruction of this object. class IsMatch { - const CodeGenInstruction *EVEXInst; + const CodeGenInstruction *OldInst; public: - IsMatch(const CodeGenInstruction *EVEXInst) : EVEXInst(EVEXInst) {} - - bool operator()(const CodeGenInstruction *VEXInst) { - RecognizableInstrBase VEXRI(*VEXInst); - RecognizableInstrBase EVEXRI(*EVEXInst); - bool VEX_W = VEXRI.HasREX_W; - bool EVEX_W = EVEXRI.HasREX_W; - bool VEX_WIG = VEXRI.IgnoresW; - bool EVEX_WIG = EVEXRI.IgnoresW; - bool EVEX_W1_VEX_W0 = EVEXInst->TheDef->getValueAsBit("EVEX_W1_VEX_W0"); - - if (VEXRI.IsCodeGenOnly != EVEXRI.IsCodeGenOnly || - // VEX/EVEX fields - VEXRI.OpPrefix != EVEXRI.OpPrefix || VEXRI.OpMap != EVEXRI.OpMap || - VEXRI.HasVEX_4V != EVEXRI.HasVEX_4V || - VEXRI.HasVEX_L != EVEXRI.HasVEX_L || - // Match is allowed if either is VEX_WIG, or they match, or EVEX - // is VEX_W1X and VEX is VEX_W0. - (!(VEX_WIG || (!EVEX_WIG && EVEX_W == VEX_W) || - (EVEX_W1_VEX_W0 && EVEX_W && !VEX_W))) || - // Instruction's format - VEXRI.Form != EVEXRI.Form) + IsMatch(const CodeGenInstruction *OldInst) : OldInst(OldInst) {} + + bool operator()(const CodeGenInstruction *NewInst) { + RecognizableInstrBase NewRI(*NewInst); + RecognizableInstrBase OldRI(*OldInst); + + // Return false if any of the following fields of does not match. + if (std::make_tuple(OldRI.IsCodeGenOnly, OldRI.OpMap, NewRI.OpPrefix, + OldRI.HasVEX_4V, OldRI.HasVEX_L, OldRI.HasREX_W, + OldRI.Form) != + std::make_tuple(NewRI.IsCodeGenOnly, NewRI.OpMap, OldRI.OpPrefix, + NewRI.HasVEX_4V, NewRI.HasVEX_L, NewRI.HasREX_W, + NewRI.Form)) return false; // This is needed for instructions with intrinsic version (_Int). @@ -131,9 +120,9 @@ public: // Also for instructions that their EVEX version was upgraded to work with // k-registers. For example VPCMPEQBrm (xmm output register) and // VPCMPEQBZ128rm (k register output register). - for (unsigned i = 0, e = EVEXInst->Operands.size(); i < e; i++) { - Record *OpRec1 = EVEXInst->Operands[i].Rec; - Record *OpRec2 = VEXInst->Operands[i].Rec; + for (unsigned i = 0, e = OldInst->Operands.size(); i < e; i++) { + Record *OpRec1 = OldInst->Operands[i].Rec; + Record *OpRec2 = NewInst->Operands[i].Rec; if (OpRec1 == OpRec2) continue; diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp index 101b75e2f087..8a860d0945bb 100644 --- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -374,8 +374,7 @@ public: RegRI.HasEVEX_L2, RegRI.HasEVEX_NF, RegRec->getValueAsBit("hasEVEX_RC"), RegRec->getValueAsBit("hasLockPrefix"), - RegRec->getValueAsBit("hasNoTrackPrefix"), - RegRec->getValueAsBit("EVEX_W1_VEX_W0")) != + RegRec->getValueAsBit("hasNoTrackPrefix")) != std::make_tuple(MemRI.Encoding, MemRI.Opcode, MemRI.OpPrefix, MemRI.OpMap, MemRI.OpSize, MemRI.AdSize, MemRI.HasREX_W, MemRI.HasVEX_4V, MemRI.HasVEX_L, MemRI.IgnoresVEX_L, @@ -383,8 +382,7 @@ public: MemRI.HasEVEX_L2, MemRI.HasEVEX_NF, MemRec->getValueAsBit("hasEVEX_RC"), MemRec->getValueAsBit("hasLockPrefix"), - MemRec->getValueAsBit("hasNoTrackPrefix"), - MemRec->getValueAsBit("EVEX_W1_VEX_W0"))) + MemRec->getValueAsBit("hasNoTrackPrefix"))) return false; // Make sure the sizes of the operands of both instructions suit each other. diff --git a/llvm/utils/TableGen/X86ManualCompressEVEXTables.def b/llvm/utils/TableGen/X86ManualCompressEVEXTables.def index 0da32f92502c..58ca10e9e10f 100644 --- a/llvm/utils/TableGen/X86ManualCompressEVEXTables.def +++ b/llvm/utils/TableGen/X86ManualCompressEVEXTables.def @@ -85,4 +85,247 @@ ENTRY(VSHUFI32X4Z256rmi, VPERM2I128rm) ENTRY(VSHUFI32X4Z256rri, VPERM2I128rr) ENTRY(VSHUFI64X2Z256rmi, VPERM2I128rm) ENTRY(VSHUFI64X2Z256rri, VPERM2I128rr) +// W bit does not match +ENTRY(VADDPDZ128rm, VADDPDrm) +ENTRY(VADDPDZ128rr, VADDPDrr) +ENTRY(VADDSDZrm, VADDSDrm) +ENTRY(VADDSDZrm_Int, VADDSDrm_Int) +ENTRY(VADDSDZrr, VADDSDrr) +ENTRY(VADDSDZrr_Int, VADDSDrr_Int) +ENTRY(VANDNPDZ128rm, VANDNPDrm) +ENTRY(VANDNPDZ128rr, VANDNPDrr) +ENTRY(VANDPDZ128rm, VANDPDrm) +ENTRY(VANDPDZ128rr, VANDPDrr) +ENTRY(VCOMISDZrm, VCOMISDrm) +ENTRY(VCOMISDZrm_Int, VCOMISDrm_Int) +ENTRY(VCOMISDZrr, VCOMISDrr) +ENTRY(VCOMISDZrr_Int, VCOMISDrr_Int) +ENTRY(VCVTPD2DQZ128rm, VCVTPD2DQrm) +ENTRY(VCVTPD2DQZ128rr, VCVTPD2DQrr) +ENTRY(VCVTPD2PSZ128rm, VCVTPD2PSrm) +ENTRY(VCVTPD2PSZ128rr, VCVTPD2PSrr) +ENTRY(VCVTSD2SSZrm, VCVTSD2SSrm) +ENTRY(VCVTSD2SSZrm_Int, VCVTSD2SSrm_Int) +ENTRY(VCVTSD2SSZrr, VCVTSD2SSrr) +ENTRY(VCVTSD2SSZrr_Int, VCVTSD2SSrr_Int) +ENTRY(VCVTTPD2DQZ128rm, VCVTTPD2DQrm) +ENTRY(VCVTTPD2DQZ128rr, VCVTTPD2DQrr) +ENTRY(VDIVPDZ128rm, VDIVPDrm) +ENTRY(VDIVPDZ128rr, VDIVPDrr) +ENTRY(VDIVSDZrm, VDIVSDrm) +ENTRY(VDIVSDZrm_Int, VDIVSDrm_Int) +ENTRY(VDIVSDZrr, VDIVSDrr) +ENTRY(VDIVSDZrr_Int, VDIVSDrr_Int) +ENTRY(VMAXCPDZ128rm, VMAXCPDrm) +ENTRY(VMAXCPDZ128rr, VMAXCPDrr) +ENTRY(VMAXCSDZrm, VMAXCSDrm) +ENTRY(VMAXCSDZrr, VMAXCSDrr) +ENTRY(VMAXPDZ128rm, VMAXPDrm) +ENTRY(VMAXPDZ128rr, VMAXPDrr) +ENTRY(VMAXSDZrm_Int, VMAXSDrm_Int) +ENTRY(VMAXSDZrr_Int, VMAXSDrr_Int) +ENTRY(VMINCPDZ128rm, VMINCPDrm) +ENTRY(VMINCPDZ128rr, VMINCPDrr) +ENTRY(VMINCSDZrm, VMINCSDrm) +ENTRY(VMINCSDZrr, VMINCSDrr) +ENTRY(VMINPDZ128rm, VMINPDrm) +ENTRY(VMINPDZ128rr, VMINPDrr) +ENTRY(VMINSDZrm_Int, VMINSDrm_Int) +ENTRY(VMINSDZrr_Int, VMINSDrr_Int) +ENTRY(VMOVAPDZ128mr, VMOVAPDmr) +ENTRY(VMOVAPDZ128rm, VMOVAPDrm) +ENTRY(VMOVAPDZ128rr, VMOVAPDrr) +ENTRY(VMOVDDUPZ128rm, VMOVDDUPrm) +ENTRY(VMOVDDUPZ128rr, VMOVDDUPrr) +ENTRY(VMOVDQA64Z128mr, VMOVDQAmr) +ENTRY(VMOVDQA64Z128rm, VMOVDQArm) +ENTRY(VMOVDQA64Z128rr, VMOVDQArr) +ENTRY(VMOVDQU64Z128mr, VMOVDQUmr) +ENTRY(VMOVDQU64Z128rm, VMOVDQUrm) +ENTRY(VMOVDQU64Z128rr, VMOVDQUrr) +ENTRY(VMOVHPDZ128mr, VMOVHPDmr) +ENTRY(VMOVHPDZ128rm, VMOVHPDrm) +ENTRY(VMOVLPDZ128mr, VMOVLPDmr) +ENTRY(VMOVLPDZ128rm, VMOVLPDrm) +ENTRY(VMOVNTPDZ128mr, VMOVNTPDmr) +ENTRY(VMOVPQI2QIZmr, VMOVPQI2QImr) +ENTRY(VMOVPQI2QIZrr, VMOVPQI2QIrr) +ENTRY(VMOVQI2PQIZrm, VMOVQI2PQIrm) +ENTRY(VMOVSDZmr, VMOVSDmr) +ENTRY(VMOVSDZrm, VMOVSDrm) +ENTRY(VMOVSDZrm_alt, VMOVSDrm_alt) +ENTRY(VMOVSDZrr, VMOVSDrr) +ENTRY(VMOVUPDZ128mr, VMOVUPDmr) +ENTRY(VMOVUPDZ128rm, VMOVUPDrm) +ENTRY(VMOVUPDZ128rr, VMOVUPDrr) +ENTRY(VMOVZPQILo2PQIZrr, VMOVZPQILo2PQIrr) +ENTRY(VMULPDZ128rm, VMULPDrm) +ENTRY(VMULPDZ128rr, VMULPDrr) +ENTRY(VMULSDZrm, VMULSDrm) +ENTRY(VMULSDZrm_Int, VMULSDrm_Int) +ENTRY(VMULSDZrr, VMULSDrr) +ENTRY(VMULSDZrr_Int, VMULSDrr_Int) +ENTRY(VORPDZ128rm, VORPDrm) +ENTRY(VORPDZ128rr, VORPDrr) +ENTRY(VPADDQZ128rm, VPADDQrm) +ENTRY(VPADDQZ128rr, VPADDQrr) +ENTRY(VPANDNQZ128rm, VPANDNrm) +ENTRY(VPANDNQZ128rr, VPANDNrr) +ENTRY(VPANDQZ128rm, VPANDrm) +ENTRY(VPANDQZ128rr, VPANDrr) +ENTRY(VPERMILPDZ128mi, VPERMILPDmi) +ENTRY(VPERMILPDZ128ri, VPERMILPDri) +ENTRY(VPERMILPDZ128rm, VPERMILPDrm) +ENTRY(VPERMILPDZ128rr, VPERMILPDrr) +ENTRY(VPMULDQZ128rm, VPMULDQrm) +ENTRY(VPMULDQZ128rr, VPMULDQrr) +ENTRY(VPMULUDQZ128rm, VPMULUDQrm) +ENTRY(VPMULUDQZ128rr, VPMULUDQrr) +ENTRY(VPORQZ128rm, VPORrm) +ENTRY(VPORQZ128rr, VPORrr) +ENTRY(VPSLLQZ128ri, VPSLLQri) +ENTRY(VPSLLQZ128rm, VPSLLQrm) +ENTRY(VPSLLQZ128rr, VPSLLQrr) +ENTRY(VPSRLQZ128ri, VPSRLQri) +ENTRY(VPSRLQZ128rm, VPSRLQrm) +ENTRY(VPSRLQZ128rr, VPSRLQrr) +ENTRY(VPSUBQZ128rm, VPSUBQrm) +ENTRY(VPSUBQZ128rr, VPSUBQrr) +ENTRY(VPUNPCKHQDQZ128rm, VPUNPCKHQDQrm) +ENTRY(VPUNPCKHQDQZ128rr, VPUNPCKHQDQrr) +ENTRY(VPUNPCKLQDQZ128rm, VPUNPCKLQDQrm) +ENTRY(VPUNPCKLQDQZ128rr, VPUNPCKLQDQrr) +ENTRY(VPXORQZ128rm, VPXORrm) +ENTRY(VPXORQZ128rr, VPXORrr) +ENTRY(VRNDSCALEPDZ128rmi, VROUNDPDm) +ENTRY(VRNDSCALEPDZ128rri, VROUNDPDr) +ENTRY(VRNDSCALESDZm, VROUNDSDm) +ENTRY(VRNDSCALESDZm_Int, VROUNDSDm_Int) +ENTRY(VRNDSCALESDZr, VROUNDSDr) +ENTRY(VRNDSCALESDZr_Int, VROUNDSDr_Int) +ENTRY(VSHUFPDZ128rmi, VSHUFPDrmi) +ENTRY(VSHUFPDZ128rri, VSHUFPDrri) +ENTRY(VSQRTPDZ128m, VSQRTPDm) +ENTRY(VSQRTPDZ128r, VSQRTPDr) +ENTRY(VSQRTSDZm, VSQRTSDm) +ENTRY(VSQRTSDZm_Int, VSQRTSDm_Int) +ENTRY(VSQRTSDZr, VSQRTSDr) +ENTRY(VSQRTSDZr_Int, VSQRTSDr_Int) +ENTRY(VSUBPDZ128rm, VSUBPDrm) +ENTRY(VSUBPDZ128rr, VSUBPDrr) +ENTRY(VSUBSDZrm, VSUBSDrm) +ENTRY(VSUBSDZrm_Int, VSUBSDrm_Int) +ENTRY(VSUBSDZrr, VSUBSDrr) +ENTRY(VSUBSDZrr_Int, VSUBSDrr_Int) +ENTRY(VUCOMISDZrm, VUCOMISDrm) +ENTRY(VUCOMISDZrm_Int, VUCOMISDrm_Int) +ENTRY(VUCOMISDZrr, VUCOMISDrr) +ENTRY(VUCOMISDZrr_Int, VUCOMISDrr_Int) +ENTRY(VUNPCKHPDZ128rm, VUNPCKHPDrm) +ENTRY(VUNPCKHPDZ128rr, VUNPCKHPDrr) +ENTRY(VUNPCKLPDZ128rm, VUNPCKLPDrm) +ENTRY(VUNPCKLPDZ128rr, VUNPCKLPDrr) +ENTRY(VXORPDZ128rm, VXORPDrm) +ENTRY(VXORPDZ128rr, VXORPDrr) +ENTRY(VADDPDZ256rm, VADDPDYrm) +ENTRY(VADDPDZ256rr, VADDPDYrr) +ENTRY(VANDNPDZ256rm, VANDNPDYrm) +ENTRY(VANDNPDZ256rr, VANDNPDYrr) +ENTRY(VANDPDZ256rm, VANDPDYrm) +ENTRY(VANDPDZ256rr, VANDPDYrr) +ENTRY(VCVTPD2DQZ256rm, VCVTPD2DQYrm) +ENTRY(VCVTPD2DQZ256rr, VCVTPD2DQYrr) +ENTRY(VCVTPD2PSZ256rm, VCVTPD2PSYrm) +ENTRY(VCVTPD2PSZ256rr, VCVTPD2PSYrr) +ENTRY(VCVTTPD2DQZ256rm, VCVTTPD2DQYrm) +ENTRY(VCVTTPD2DQZ256rr, VCVTTPD2DQYrr) +ENTRY(VDIVPDZ256rm, VDIVPDYrm) +ENTRY(VDIVPDZ256rr, VDIVPDYrr) +ENTRY(VEXTRACTF64x2Z256mr, VEXTRACTF128mr) +ENTRY(VEXTRACTF64x2Z256rr, VEXTRACTF128rr) +ENTRY(VEXTRACTI64x2Z256mr, VEXTRACTI128mr) +ENTRY(VEXTRACTI64x2Z256rr, VEXTRACTI128rr) +ENTRY(VINSERTF64x2Z256rm, VINSERTF128rm) +ENTRY(VINSERTF64x2Z256rr, VINSERTF128rr) +ENTRY(VINSERTI64x2Z256rm, VINSERTI128rm) +ENTRY(VINSERTI64x2Z256rr, VINSERTI128rr) +ENTRY(VMAXCPDZ256rm, VMAXCPDYrm) +ENTRY(VMAXCPDZ256rr, VMAXCPDYrr) +ENTRY(VMAXPDZ256rm, VMAXPDYrm) +ENTRY(VMAXPDZ256rr, VMAXPDYrr) +ENTRY(VMINCPDZ256rm, VMINCPDYrm) +ENTRY(VMINCPDZ256rr, VMINCPDYrr) +ENTRY(VMINPDZ256rm, VMINPDYrm) +ENTRY(VMINPDZ256rr, VMINPDYrr) +ENTRY(VMOVAPDZ256mr, VMOVAPDYmr) +ENTRY(VMOVAPDZ256rm, VMOVAPDYrm) +ENTRY(VMOVAPDZ256rr, VMOVAPDYrr) +ENTRY(VMOVDDUPZ256rm, VMOVDDUPYrm) +ENTRY(VMOVDDUPZ256rr, VMOVDDUPYrr) +ENTRY(VMOVDQA64Z256mr, VMOVDQAYmr) +ENTRY(VMOVDQA64Z256rm, VMOVDQAYrm) +ENTRY(VMOVDQA64Z256rr, VMOVDQAYrr) +ENTRY(VMOVDQU64Z256mr, VMOVDQUYmr) +ENTRY(VMOVDQU64Z256rm, VMOVDQUYrm) +ENTRY(VMOVDQU64Z256rr, VMOVDQUYrr) +ENTRY(VMOVNTPDZ256mr, VMOVNTPDYmr) +ENTRY(VMOVUPDZ256mr, VMOVUPDYmr) +ENTRY(VMOVUPDZ256rm, VMOVUPDYrm) +ENTRY(VMOVUPDZ256rr, VMOVUPDYrr) +ENTRY(VMULPDZ256rm, VMULPDYrm) +ENTRY(VMULPDZ256rr, VMULPDYrr) +ENTRY(VORPDZ256rm, VORPDYrm) +ENTRY(VORPDZ256rr, VORPDYrr) +ENTRY(VPADDQZ256rm, VPADDQYrm) +ENTRY(VPADDQZ256rr, VPADDQYrr) +ENTRY(VPANDNQZ256rm, VPANDNYrm) +ENTRY(VPANDNQZ256rr, VPANDNYrr) +ENTRY(VPANDQZ256rm, VPANDYrm) +ENTRY(VPANDQZ256rr, VPANDYrr) +ENTRY(VPERMILPDZ256mi, VPERMILPDYmi) +ENTRY(VPERMILPDZ256ri, VPERMILPDYri) +ENTRY(VPERMILPDZ256rm, VPERMILPDYrm) +ENTRY(VPERMILPDZ256rr, VPERMILPDYrr) +ENTRY(VPMULDQZ256rm, VPMULDQYrm) +ENTRY(VPMULDQZ256rr, VPMULDQYrr) +ENTRY(VPMULUDQZ256rm, VPMULUDQYrm) +ENTRY(VPMULUDQZ256rr, VPMULUDQYrr) +ENTRY(VPORQZ256rm, VPORYrm) +ENTRY(VPORQZ256rr, VPORYrr) +ENTRY(VPSLLQZ256ri, VPSLLQYri) +ENTRY(VPSLLQZ256rm, VPSLLQYrm) +ENTRY(VPSLLQZ256rr, VPSLLQYrr) +ENTRY(VPSRLQZ256ri, VPSRLQYri) +ENTRY(VPSRLQZ256rm, VPSRLQYrm) +ENTRY(VPSRLQZ256rr, VPSRLQYrr) +ENTRY(VPSUBQZ256rm, VPSUBQYrm) +ENTRY(VPSUBQZ256rr, VPSUBQYrr) +ENTRY(VPUNPCKHQDQZ256rm, VPUNPCKHQDQYrm) +ENTRY(VPUNPCKHQDQZ256rr, VPUNPCKHQDQYrr) +ENTRY(VPUNPCKLQDQZ256rm, VPUNPCKLQDQYrm) +ENTRY(VPUNPCKLQDQZ256rr, VPUNPCKLQDQYrr) +ENTRY(VPXORQZ256rm, VPXORYrm) +ENTRY(VPXORQZ256rr, VPXORYrr) +ENTRY(VRNDSCALEPDZ256rmi, VROUNDPDYm) +ENTRY(VRNDSCALEPDZ256rri, VROUNDPDYr) +ENTRY(VSHUFPDZ256rmi, VSHUFPDYrmi) +ENTRY(VSHUFPDZ256rri, VSHUFPDYrri) +ENTRY(VSQRTPDZ256m, VSQRTPDYm) +ENTRY(VSQRTPDZ256r, VSQRTPDYr) +ENTRY(VSUBPDZ256rm, VSUBPDYrm) +ENTRY(VSUBPDZ256rr, VSUBPDYrr) +ENTRY(VUNPCKHPDZ256rm, VUNPCKHPDYrm) +ENTRY(VUNPCKHPDZ256rr, VUNPCKHPDYrr) +ENTRY(VUNPCKLPDZ256rm, VUNPCKLPDYrm) +ENTRY(VUNPCKLPDZ256rr, VUNPCKLPDYrr) +ENTRY(VXORPDZ256rm, VXORPDYrm) +ENTRY(VXORPDZ256rr, VXORPDYrr) +ENTRY(VPBROADCASTQZ128rm, VPBROADCASTQrm) +ENTRY(VPBROADCASTQZ128rr, VPBROADCASTQrr) +ENTRY(VBROADCASTF64X2Z128rm, VBROADCASTF128rm) +ENTRY(VBROADCASTI64X2Z128rm, VBROADCASTI128rm) +ENTRY(VBROADCASTSDZ256rm, VBROADCASTSDYrm) +ENTRY(VBROADCASTSDZ256rr, VBROADCASTSDYrr) +ENTRY(VPBROADCASTQZ256rm, VPBROADCASTQYrm) +ENTRY(VPBROADCASTQZ256rr, VPBROADCASTQYrr) #undef ENTRY -- GitLab From b2246cf73e10c38aefffd923e4b53a1975f45909 Mon Sep 17 00:00:00 2001 From: Haojian Wu Date: Sat, 6 Jan 2024 10:18:27 +0100 Subject: [PATCH 192/728] Bazel port for a5902a4d2425ac083f1530719e35b5c562cb1e60 --- utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 6d162300169c..1110daa62a1c 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -2187,7 +2187,7 @@ llvm_target_lib_list = [lib for lib in [ ("-gen-callingconv", "lib/Target/X86/X86GenCallingConv.inc"), ("-gen-subtarget", "lib/Target/X86/X86GenSubtargetInfo.inc"), ("-gen-x86-fold-tables -asmwriternum=1", "lib/Target/X86/X86GenFoldTables.inc"), - ("-gen-x86-EVEX2VEX-tables", "lib/Target/X86/X86GenEVEX2VEXTables.inc"), + ("-gen-x86-compress-evex-tables", "lib/Target/X86/X86GenCompressEVEXTables.inc"), ("-gen-exegesis", "lib/Target/X86/X86GenExegesis.inc"), ("-gen-x86-mnemonic-tables -asmwriternum=1", "lib/Target/X86/X86GenMnemonicTables.inc"), ], -- GitLab From ab073cbccb6e79d8b65a286e8948bc1f07c7c09b Mon Sep 17 00:00:00 2001 From: Shubham Sandeep Rastogi Date: Sat, 6 Jan 2024 01:34:20 -0800 Subject: [PATCH 193/728] Add requires darwin to verify-no-file.yaml (#77188) --- llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml b/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml index 1327cc26ee9f..808cc7be638b 100644 --- a/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml +++ b/llvm/test/tools/llvm-dwarfdump/AArch64/verify-no-file.yaml @@ -1,5 +1,6 @@ # RUN: yaml2obj %s -o %t.o # RUN: llvm-dwarfdump -arch arm64 --debug-line --verify %t.o 2>&1 | FileCheck %s +# REQUIRES: system-darwin # CHECK-NOT: error: .debug_line[0x{{[0-9a-f]+}}][0] has invalid file index 1 (valid values are [1,0]): --- !mach-o -- GitLab From 5b33cff39753c790ecc6847435664592abe40415 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Sat, 6 Jan 2024 11:17:01 +0100 Subject: [PATCH 194/728] [mlir][gpu] Add Support for Cluster of Thread Blocks in `gpu.launch` (#76924) --- mlir/include/mlir/Dialect/GPU/IR/GPUOps.td | 53 +++++++++-- mlir/lib/Dialect/GPU/IR/GPUDialect.cpp | 88 +++++++++++++++++-- .../GPU/Transforms/KernelOutlining.cpp | 20 +++-- .../SCFToGPU/no_blocks_no_threads.mlir | 4 +- mlir/test/Dialect/GPU/invalid.mlir | 2 +- mlir/test/Dialect/GPU/outlining.mlir | 74 ++++++++++++++++ 6 files changed, 219 insertions(+), 22 deletions(-) diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td index efef61b5c6e7..8d4a110ee801 100644 --- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td +++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td @@ -678,6 +678,9 @@ def GPU_LaunchOp : GPU_Op<"launch", [ Arguments<(ins Variadic:$asyncDependencies, Index:$gridSizeX, Index:$gridSizeY, Index:$gridSizeZ, Index:$blockSizeX, Index:$blockSizeY, Index:$blockSizeZ, + Optional:$clusterSizeX, + Optional:$clusterSizeY, + Optional:$clusterSizeZ, Optional:$dynamicSharedMemorySize)>, Results<(outs Optional:$asyncToken)> { let summary = "GPU kernel launch operation"; @@ -700,8 +703,11 @@ def GPU_LaunchOp : GPU_Op<"launch", [ to the amount of dynamic shared memory a kernel's workgroup should be allocated; when this operand is not present, a zero size is assumed. - The body region has at least _twelve_ arguments, grouped as follows: + The body region has at least _twelve_ arguments, or _eighteen_ if cluster + dimensions are present, grouped as follows: + - three optional arguments that contain cluster identifiers along x,y,z + dimensions; - three arguments that contain block identifiers along x,y,z dimensions; - three arguments that contain thread identifiers along x,y,z dimensions; - operands of the `gpu.launch` operation as is (i.e. the operands for @@ -713,6 +719,7 @@ def GPU_LaunchOp : GPU_Op<"launch", [ ``` operation ::= `gpu.launch` (`async` (`[` ssa-id-list `]`)? )? + ( `clusters` `(` ssa-id-list `)` `in` ssa-reassignment )? `blocks` `(` ssa-id-list `)` `in` ssa-reassignment `threads` `(` ssa-id-list `)` `in` ssa-reassignment (dynamic_shared_memory_size ssa-use)? @@ -763,6 +770,16 @@ def GPU_LaunchOp : GPU_Op<"launch", [ // Assuming %val1 is defined outside the gpu.launch region. %42 = load %workgroup[%bx] : memref<32xf32, 3> } + + // Launch with clusters. + gpu.launch clusters(%cx, %cy, %cz) in (%sz_cx = %0, %sz_cy = %1, %sz_cz = %2) + blocks(%bx, %by, %bz) in (%sz_bx = %3, %sz_by = %4, %sz_bz = %5) + threads(%tx, %ty, %tz) in (%sz_tx = %6, %sz_ty = %7, %sz_tz = %8) + { + // Cluster, block and thread identifiers, as well as cluster/block/grid + // sizes are immediately usable inside body region. + "some_op"(%cx, %bx, %tx) : (index, index, index) -> () + } ``` Rationale: using operation/block arguments gives analyses a clear way of @@ -784,7 +801,10 @@ def GPU_LaunchOp : GPU_Op<"launch", [ CArg<"Type", "nullptr">:$asyncTokenType, CArg<"ValueRange", "{}">:$asyncDependencies, CArg<"TypeRange", "{}">:$workgroupAttributions, - CArg<"TypeRange", "{}">:$privateAttributions)> + CArg<"TypeRange", "{}">:$privateAttributions, + CArg<"Value", "nullptr">:$clusterSizeX, + CArg<"Value", "nullptr">:$clusterSizeY, + CArg<"Value", "nullptr">:$clusterSizeZ)> ]; let extraClassDeclaration = [{ @@ -792,17 +812,24 @@ def GPU_LaunchOp : GPU_Op<"launch", [ KernelDim3 getBlockIds(); /// Get the SSA values corresponding to kernel thread identifiers. KernelDim3 getThreadIds(); + /// Get the SSA values corresponding to kernel cluster identifiers. + std::optional getClusterIds(); /// Get the SSA values corresponding to kernel grid size. KernelDim3 getGridSize(); /// Get the SSA values corresponding to kernel block size. KernelDim3 getBlockSize(); + /// Get the SSA values corresponding to kernel cluster size. + std::optional getClusterSize(); /// Get the SSA values passed as operands to specify the grid size. KernelDim3 getGridSizeOperandValues(); /// Get the SSA values passed as operands to specify the block size. KernelDim3 getBlockSizeOperandValues(); + /// Get the SSA values passed as operands to specify the cluster size. + std::optional getClusterSizeOperandValues(); static StringRef getBlocksKeyword() { return "blocks"; } + static StringRef getClustersKeyword() { return "clusters"; } static StringRef getThreadsKeyword() { return "threads"; } static StringRef getDynamicSharedMemorySizeKeyword() { return "dynamic_shared_memory_size"; @@ -816,6 +843,21 @@ def GPU_LaunchOp : GPU_Op<"launch", [ /// placed in the leading positions of the argument list. static constexpr unsigned kNumConfigRegionAttributes = 12; + /// Returns true if cluster size is specified. + bool hasClusterSize() { + if (getClusterSizeX() && getClusterSizeY() && getClusterSizeZ()) + return true; + return false; + } + /// Returns the number of operands including cluster size + unsigned getNumConfigOperands() { + return kNumConfigOperands + (hasClusterSize() ? 3 : 0); + } + /// Returns the number of region attributes including cluster size + unsigned getNumConfigRegionAttributes() { + return kNumConfigRegionAttributes + (hasClusterSize() ? 6 : 0); + } + /// Returns the keywords used in the custom syntax for this Op. static StringRef getWorkgroupKeyword() { return "workgroup"; } static StringRef getPrivateKeyword() { return "private"; } @@ -831,7 +873,7 @@ def GPU_LaunchOp : GPU_Op<"launch", [ /// the workgroup memory ArrayRef getWorkgroupAttributions() { auto begin = - std::next(getBody().args_begin(), kNumConfigRegionAttributes); + std::next(getBody().args_begin(), getNumConfigRegionAttributes()); auto end = std::next(begin, getNumWorkgroupAttributions()); return {begin, end}; } @@ -842,7 +884,7 @@ def GPU_LaunchOp : GPU_Op<"launch", [ /// Returns the number of buffers located in the private memory. unsigned getNumPrivateAttributions() { - return getBody().getNumArguments() - kNumConfigRegionAttributes - + return getBody().getNumArguments() - getNumConfigRegionAttributes() - getNumWorkgroupAttributions(); } @@ -853,7 +895,7 @@ def GPU_LaunchOp : GPU_Op<"launch", [ // memory. auto begin = std::next(getBody().args_begin(), - kNumConfigRegionAttributes + getNumWorkgroupAttributions()); + getNumConfigRegionAttributes() + getNumWorkgroupAttributions()); return {begin, getBody().args_end()}; } @@ -871,6 +913,7 @@ def GPU_LaunchOp : GPU_Op<"launch", [ let hasCanonicalizer = 1; let hasCustomAssemblyFormat = 1; let hasRegionVerifier = 1; + let hasVerifier = 1; } def GPU_PrintfOp : GPU_Op<"printf", [MemoryEffects<[MemWrite]>]>, diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp index dd482f305fcb..020900934c9f 100644 --- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp +++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp @@ -646,7 +646,8 @@ void LaunchOp::build(OpBuilder &builder, OperationState &result, Value getBlockSizeZ, Value dynamicSharedMemorySize, Type asyncTokenType, ValueRange asyncDependencies, TypeRange workgroupAttributions, - TypeRange privateAttributions) { + TypeRange privateAttributions, Value clusterSizeX, + Value clusterSizeY, Value clusterSizeZ) { // Add a WorkGroup attribution attribute. This attribute is required to // identify private attributions in the list of block argguments. result.addAttribute(getNumWorkgroupAttributionsAttrName(), @@ -660,6 +661,12 @@ void LaunchOp::build(OpBuilder &builder, OperationState &result, // Add grid and block sizes as op operands, followed by the data operands. result.addOperands({gridSizeX, gridSizeY, gridSizeZ, getBlockSizeX, getBlockSizeY, getBlockSizeZ}); + if (clusterSizeX) + result.addOperands(clusterSizeX); + if (clusterSizeY) + result.addOperands(clusterSizeY); + if (clusterSizeZ) + result.addOperands(clusterSizeZ); if (dynamicSharedMemorySize) result.addOperands(dynamicSharedMemorySize); @@ -678,9 +685,12 @@ void LaunchOp::build(OpBuilder &builder, OperationState &result, body->addArgument(argTy, result.location); kernelRegion->push_back(body); // Fill OperandSegmentSize Attribute. - SmallVector segmentSizes(8, 1); + SmallVector segmentSizes(11, 1); segmentSizes.front() = asyncDependencies.size(); segmentSizes.back() = dynamicSharedMemorySize ? 1 : 0; + segmentSizes[7] = clusterSizeX ? 1 : 0; + segmentSizes[8] = clusterSizeY ? 1 : 0; + segmentSizes[9] = clusterSizeZ ? 1 : 0; result.addAttribute(getOperandSegmentSizeAttr(), builder.getDenseI32ArrayAttr(segmentSizes)); } @@ -709,6 +719,22 @@ KernelDim3 LaunchOp::getBlockSize() { return KernelDim3{args[9], args[10], args[11]}; } +std::optional LaunchOp::getClusterIds() { + assert(!getBody().empty() && "LaunchOp body must not be empty."); + if (!hasClusterSize()) + return std::nullopt; + auto args = getBody().getArguments(); + return KernelDim3{args[12], args[13], args[14]}; +} + +std::optional LaunchOp::getClusterSize() { + assert(!getBody().empty() && "LaunchOp body must not be empty."); + if (!hasClusterSize()) + return std::nullopt; + auto args = getBody().getArguments(); + return KernelDim3{args[15], args[16], args[17]}; +} + KernelDim3 LaunchOp::getGridSizeOperandValues() { auto operands = getOperands().drop_front(getAsyncDependencies().size()); return KernelDim3{operands[0], operands[1], operands[2]}; @@ -719,6 +745,20 @@ KernelDim3 LaunchOp::getBlockSizeOperandValues() { return KernelDim3{operands[3], operands[4], operands[5]}; } +std::optional LaunchOp::getClusterSizeOperandValues() { + auto operands = getOperands().drop_front(getAsyncDependencies().size()); + if (!hasClusterSize()) + return std::nullopt; + return KernelDim3{operands[6], operands[7], operands[8]}; +} + +LogicalResult LaunchOp::verify() { + if (!(hasClusterSize()) && + (getClusterSizeX() || getClusterSizeY() || getClusterSizeZ())) + return emitOpError() << "cluster size must be all present"; + return success(); +} + LogicalResult LaunchOp::verifyRegions() { // Kernel launch takes kNumConfigOperands leading operands for grid/block // sizes and transforms them into kNumConfigRegionAttributes region arguments @@ -778,6 +818,12 @@ void LaunchOp::print(OpAsmPrinter &p) { p << " [" << getAsyncDependencies() << ']'; } // Print the launch configuration. + if (hasClusterSize()) { + p << ' ' << getClustersKeyword(); + printSizeAssignment(p, getClusterSize().value(), + getClusterSizeOperandValues().value(), + getClusterIds().value()); + } p << ' ' << getBlocksKeyword(); printSizeAssignment(p, getGridSize(), getGridSizeOperandValues(), getBlockIds()); @@ -831,6 +877,7 @@ parseSizeAssignment(OpAsmParser &parser, /// Parses a Launch operation. /// operation ::= `gpu.launch` (`async` `[` ssa-id-list `]`)? +/// `clusters` `(` ssa-id-list `)` `in` ssa-reassignment (Optional) /// `blocks` `(` ssa-id-list `)` `in` ssa-reassignment /// `threads` `(` ssa-id-list `)` `in` ssa-reassignment /// memory-attribution @@ -840,7 +887,6 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) { // Sizes of the grid and block. SmallVector sizes(LaunchOp::kNumConfigOperands); - MutableArrayRef sizesRef(sizes); // Actual (data) operands passed to the kernel. SmallVector dataOperands; @@ -848,7 +894,6 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) { // Region arguments to be created. SmallVector regionArgs( LaunchOp::kNumConfigRegionAttributes); - MutableArrayRef regionArgsRef(regionArgs); // Parse optional async dependencies. SmallVector asyncDependencies; @@ -861,6 +906,24 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) { if (parser.getNumResults() > 0) result.types.push_back(asyncTokenType); + bool hasCluster = false; + if (succeeded( + parser.parseOptionalKeyword(LaunchOp::getClustersKeyword().data()))) { + hasCluster = true; + sizes.resize(9); + regionArgs.resize(18); + } + MutableArrayRef sizesRef(sizes); + MutableArrayRef regionArgsRef(regionArgs); + + // Last three segment assigns the cluster size. In the region argument + // list, this is last 6 arguments. + if (hasCluster) { + if (parseSizeAssignment(parser, sizesRef.drop_front(6), + regionArgsRef.slice(15, 3), + regionArgsRef.slice(12, 3))) + return failure(); + } // Parse the size assignment segments: the first segment assigns grid sizes // and defines values for block identifiers; the second segment assigns block // sizes and defines values for thread identifiers. In the region argument @@ -898,7 +961,7 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) { // LaunchOp::getNumWorkgroupAttributionsAttrName(). Type index = parser.getBuilder().getIndexType(); SmallVector dataTypes( - LaunchOp::kNumConfigRegionAttributes, index); + LaunchOp::kNumConfigRegionAttributes + 6, index); SmallVector regionArguments; for (auto ssaValueAndType : llvm::zip(regionArgs, dataTypes)) { @@ -916,8 +979,9 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) { // Store the number of operands we just parsed as the number of workgroup // memory attributions. - unsigned numWorkgroupAttrs = - regionArguments.size() - LaunchOp::kNumConfigRegionAttributes; + unsigned numWorkgroupAttrs = regionArguments.size() - + LaunchOp::kNumConfigRegionAttributes - + (hasCluster ? 6 : 0); result.addAttribute(LaunchOp::getNumWorkgroupAttributionsAttrName(), builder.getI64IntegerAttr(numWorkgroupAttrs)); @@ -934,8 +998,14 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) { parser.parseOptionalAttrDict(result.attributes)) return failure(); - SmallVector segmentSizes(8, 1); + SmallVector segmentSizes(11, 1); segmentSizes.front() = asyncDependencies.size(); + + if (!hasCluster) { + segmentSizes[7] = 0; + segmentSizes[8] = 0; + segmentSizes[9] = 0; + } segmentSizes.back() = hasDynamicSharedMemorySize ? 1 : 0; result.addAttribute(LaunchOp::getOperandSegmentSizeAttr(), parser.getBuilder().getDenseI32ArrayAttr(segmentSizes)); @@ -992,7 +1062,7 @@ BlockArgument LaunchOp::addWorkgroupAttribution(Type type, Location loc) { (*this)->setAttr(attrName, IntegerAttr::get(attr.getType(), attr.getValue() + 1)); return getBody().insertArgument( - LaunchOp::kNumConfigRegionAttributes + attr.getInt(), type, loc); + LaunchOp::getNumConfigRegionAttributes() + attr.getInt(), type, loc); } /// Adds a new block argument that corresponds to buffers located in diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp index 7432a58f18b4..2436113dc423 100644 --- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp +++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp @@ -49,15 +49,21 @@ static void createForAllDimensions(OpBuilder &builder, Location loc, /// entry block of `launchOpBody`, to the corresponding result value of the /// added operations. static void injectGpuIndexOperations(Location loc, Region &launchFuncOpBody, - Region &launchOpBody, IRMapping &map) { + Region &launchOpBody, IRMapping &map, + bool hasCluster = false) { OpBuilder builder(loc->getContext()); Block &firstBlock = launchOpBody.front(); builder.setInsertionPointToStart(&launchFuncOpBody.front()); - SmallVector indexOps; + SmallVector indexOps; + // The order is important here, as it must match the order of the arguments createForAllDimensions(builder, loc, indexOps); createForAllDimensions(builder, loc, indexOps); createForAllDimensions(builder, loc, indexOps); createForAllDimensions(builder, loc, indexOps); + if (hasCluster) { + createForAllDimensions(builder, loc, indexOps); + createForAllDimensions(builder, loc, indexOps); + } // Replace the leading 12 function args with the respective thread/block index // operations. Iterate backwards since args are erased and indices change. for (const auto &indexOp : enumerate(indexOps)) @@ -212,9 +218,11 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp, IRMapping map; // Map the arguments corresponding to the launch parameters like blockIdx, - // threadIdx, etc. + // threadIdx, etc. If cluster is present, then we also generate clusterIdx and + // clusterDim. Region &outlinedFuncBody = outlinedFunc.getBody(); - injectGpuIndexOperations(loc, outlinedFuncBody, launchOpBody, map); + injectGpuIndexOperations(loc, outlinedFuncBody, launchOpBody, map, + launchOp.hasClusterSize()); // Map memory attributions from the LaunOp op to the GPUFuncOp attributions. for (const auto &[launchArg, funcArg] : @@ -278,12 +286,14 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp, // The launch op has an optional dynamic shared memory size. If it doesn't // exist, we use zero. Value asyncToken = launchOp.getAsyncToken(); + std::optional clusterSize = + launchOp.getClusterSizeOperandValues(); auto launchFunc = builder.create( launchOp.getLoc(), kernelFunc, launchOp.getGridSizeOperandValues(), launchOp.getBlockSizeOperandValues(), launchOp.getDynamicSharedMemorySize(), operands, asyncToken ? asyncToken.getType() : nullptr, - launchOp.getAsyncDependencies()); + launchOp.getAsyncDependencies(), clusterSize); launchOp.replaceAllUsesWith(launchFunc); launchOp.erase(); } diff --git a/mlir/test/Conversion/SCFToGPU/no_blocks_no_threads.mlir b/mlir/test/Conversion/SCFToGPU/no_blocks_no_threads.mlir index a058365a104a..79eef8ae7eb8 100644 --- a/mlir/test/Conversion/SCFToGPU/no_blocks_no_threads.mlir +++ b/mlir/test/Conversion/SCFToGPU/no_blocks_no_threads.mlir @@ -17,8 +17,8 @@ func.func @one_d_loop(%A : memref, %B : memref) { // CHECK-BLOCKS-NEXT: %{{.*}} = arith.constant 1 : index // CHECK-BLOCKS-NEXT: %[[ONE:.*]] = arith.constant 1 : index - // CHECK-THREADS-NEXT: gpu.launch blocks(%[[B0:.*]], %[[B1:.*]], %[[B2:.*]]) in (%{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]], %{{.*}}0 = %[[ONE]]) threads(%[[T0:.*]], %[[T1:.*]], %[[T2:.*]]) in (%{{.*}} = %[[BOUND]], %{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]]) - // CHECK-BLOCKS-NEXT: gpu.launch blocks(%[[B0:.*]], %[[B1:.*]], %[[B2:.*]]) in (%{{.*}} = %[[BOUND]], %{{.*}} = %[[ONE]], %{{.*}}0 = %[[ONE]]) threads(%[[T0:.*]], %[[T1:.*]], %[[T2:.*]]) in (%{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]]) + // CHECK-THREADS-NEXT: gpu.launch blocks(%[[B0:.*]], %[[B1:.*]], %[[B2:.*]]) in (%{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]]) threads(%[[T0:.*]], %[[T1:.*]], %[[T2:.*]]) in (%{{.*}} = %[[BOUND]], %{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]]) + // CHECK-BLOCKS-NEXT: gpu.launch blocks(%[[B0:.*]], %[[B1:.*]], %[[B2:.*]]) in (%{{.*}} = %[[BOUND]], %{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]]) threads(%[[T0:.*]], %[[T1:.*]], %[[T2:.*]]) in (%{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]], %{{.*}} = %[[ONE]]) affine.for %i = 0 to 42 { // CHECK-THREADS-NEXT: %[[INDEX:.*]] = arith.addi %{{.*}}, %[[T0]] // CHECK-THREADS-NEXT: memref.load %{{.*}}[%[[INDEX]]] diff --git a/mlir/test/Dialect/GPU/invalid.mlir b/mlir/test/Dialect/GPU/invalid.mlir index 8a34d6432607..4d3a898fdd15 100644 --- a/mlir/test/Dialect/GPU/invalid.mlir +++ b/mlir/test/Dialect/GPU/invalid.mlir @@ -16,7 +16,7 @@ func.func @no_region_attrs(%sz : index) { ^bb1(%bx: index, %by: index, %bz: index, %tx: index, %ty: index, %tz: index): gpu.terminator - }) {operandSegmentSizes = array} : (index, index, index, index, index, index) -> () + }) {operandSegmentSizes = array} : (index, index, index, index, index, index) -> () return } diff --git a/mlir/test/Dialect/GPU/outlining.mlir b/mlir/test/Dialect/GPU/outlining.mlir index 8020f6dfa65b..601add9a9f91 100644 --- a/mlir/test/Dialect/GPU/outlining.mlir +++ b/mlir/test/Dialect/GPU/outlining.mlir @@ -407,3 +407,77 @@ func.func @launch_memory_attributions_1(%arg0 : memref<*xf32>) { } // CHECK-DL-LABEL: gpu.module @launch_memory_attributions_1_kernel attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry>} + +// ----- +// CHECK: module attributes {gpu.container_module} + +// CHECK-LABEL: func @launch_cluster() +func.func @launch_cluster() { + // CHECK: %[[ARG0:.*]] = "op"() : () -> f32 + %0 = "op"() : () -> (f32) + // CHECK: %[[ARG1:.*]] = "op"() : () -> memref + %1 = "op"() : () -> (memref) + // CHECK: %[[CDIMX:.*]] = arith.constant 1 + %cDimX = arith.constant 1 : index + // CHECK: %[[CDIMY:.*]] = arith.constant 2 + %cDimY = arith.constant 2 : index + // CHECK: %[[CDIMZ:.*]] = arith.constant 1 + %cDimZ = arith.constant 1 : index + // CHECK: %[[GDIMX:.*]] = arith.constant 8 + %gDimX = arith.constant 8 : index + // CHECK: %[[GDIMY:.*]] = arith.constant 12 + %gDimY = arith.constant 12 : index + // CHECK: %[[GDIMZ:.*]] = arith.constant 16 + %gDimZ = arith.constant 16 : index + // CHECK: %[[BDIMX:.*]] = arith.constant 20 + %bDimX = arith.constant 20 : index + // CHECK: %[[BDIMY:.*]] = arith.constant 24 + %bDimY = arith.constant 24 : index + // CHECK: %[[BDIMZ:.*]] = arith.constant 28 + %bDimZ = arith.constant 28 : index + + // CHECK: gpu.launch_func @launch_cluster_kernel::@launch_cluster_kernel clusters in (%[[CDIMX]], %[[CDIMY]], %[[CDIMZ]]) blocks in (%[[GDIMX]], %[[GDIMY]], %[[GDIMZ]]) threads in (%[[BDIMX]], %[[BDIMY]], %[[BDIMZ]]) args(%[[ARG0]] : f32, %[[ARG1]] : memref) + // CHECK-NOT: gpu.launch blocks + gpu.launch clusters(%cx, %cy, %cz) in (%cluster_x = %cDimX, %cluster_y = %cDimY, + %cluster_z = %cDimZ) + blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, + %grid_z = %gDimZ) + threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, + %block_z = %bDimZ) { + "use"(%0): (f32) -> () + "some_op"(%cx, %bx, %block_x) : (index, index, index) -> () + %42 = memref.load %1[%tx] : memref + gpu.terminator + } + return +} + +// CHECK-LABEL: gpu.module @launch_cluster_kernel +// CHECK-NEXT: gpu.func @launch_cluster_kernel +// CHECK-SAME: (%[[KERNEL_ARG0:.*]]: f32, %[[KERNEL_ARG1:.*]]: memref) +// CHECK-SAME: gpu.known_block_size = array +// CHECK-SAME: gpu.known_grid_size = array +// CHECK-NEXT: %[[BID:.*]] = gpu.block_id x +// CHECK-NEXT: = gpu.block_id y +// CHECK-NEXT: = gpu.block_id z +// CHECK-NEXT: %[[TID:.*]] = gpu.thread_id x +// CHECK-NEXT: = gpu.thread_id y +// CHECK-NEXT: = gpu.thread_id z +// CHECK-NEXT: = gpu.grid_dim x +// CHECK-NEXT: = gpu.grid_dim y +// CHECK-NEXT: = gpu.grid_dim z +// CHECK-NEXT: %[[BDIM:.*]] = gpu.block_dim x +// CHECK-NEXT: = gpu.block_dim y +// CHECK-NEXT: = gpu.block_dim z +// CHECK-NEXT: %[[CID:.*]] = gpu.cluster_id x +// CHECK-NEXT: = gpu.cluster_id y +// CHECK-NEXT: = gpu.cluster_id z +// CHECK-NEXT: %[[CDIM:.*]] = gpu.cluster_dim x +// CHECK-NEXT: = gpu.cluster_dim y +// CHECK-NEXT: = gpu.cluster_dim z +// CHECK-NEXT: cf.br ^[[BLOCK:.*]] +// CHECK-NEXT: ^[[BLOCK]]: +// CHECK-NEXT: "use"(%[[KERNEL_ARG0]]) : (f32) -> () +// CHECK-NEXT: "some_op"(%[[CID]], %[[BID]], %[[BDIM]]) : (index, index, index) -> () +// CHECK-NEXT: = memref.load %[[KERNEL_ARG1]][%[[TID]]] : memref + -- GitLab From 1687555572ee4fb435da400dde02e7a1e60b742c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= Date: Sat, 6 Jan 2024 11:28:10 +0100 Subject: [PATCH 195/728] [GlobalIsel] Combine select of binops (#76763) --- .../llvm/CodeGen/GlobalISel/CombinerHelper.h | 3 + .../CodeGen/GlobalISel/GenericMachineInstrs.h | 103 ++++++++++++ .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 93 +++++++---- .../AArch64/GlobalISel/combine-select.mir | 151 ++++++++++++++++++ 4 files changed, 322 insertions(+), 28 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index dcc1a4580b14..f3b68623596c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -910,6 +910,9 @@ private: bool tryFoldSelectOfConstants(GSelect *Select, BuildFnTy &MatchInfo); + /// Try to fold select(cc, binop(), binop()) -> binop(select(), X) + bool tryFoldSelectOfBinOps(GSelect *Select, BuildFnTy &MatchInfo); + bool isOneOrOneSplat(Register Src, bool AllowUndefs); bool isZeroOrZeroSplat(Register Src, bool AllowUndefs); bool isConstantSplatVector(Register Src, int64_t SplatValue, diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h index 6ab1d4550c51..21d98d30356c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h @@ -558,6 +558,109 @@ public: } }; +// Represents a binary operation, i.e, x = y op z. +class GBinOp : public GenericMachineInstr { +public: + Register getLHSReg() const { return getReg(1); } + Register getRHSReg() const { return getReg(2); } + + static bool classof(const MachineInstr *MI) { + switch (MI->getOpcode()) { + // Integer. + case TargetOpcode::G_ADD: + case TargetOpcode::G_SUB: + case TargetOpcode::G_MUL: + case TargetOpcode::G_SDIV: + case TargetOpcode::G_UDIV: + case TargetOpcode::G_SREM: + case TargetOpcode::G_UREM: + case TargetOpcode::G_SMIN: + case TargetOpcode::G_SMAX: + case TargetOpcode::G_UMIN: + case TargetOpcode::G_UMAX: + // Floating point. + case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMAXNUM: + case TargetOpcode::G_FMINNUM_IEEE: + case TargetOpcode::G_FMAXNUM_IEEE: + case TargetOpcode::G_FMINIMUM: + case TargetOpcode::G_FMAXIMUM: + case TargetOpcode::G_FADD: + case TargetOpcode::G_FSUB: + case TargetOpcode::G_FMUL: + case TargetOpcode::G_FDIV: + case TargetOpcode::G_FPOW: + // Logical. + case TargetOpcode::G_AND: + case TargetOpcode::G_OR: + case TargetOpcode::G_XOR: + return true; + default: + return false; + } + }; +}; + +// Represents an integer binary operation. +class GIntBinOp : public GBinOp { +public: + static bool classof(const MachineInstr *MI) { + switch (MI->getOpcode()) { + case TargetOpcode::G_ADD: + case TargetOpcode::G_SUB: + case TargetOpcode::G_MUL: + case TargetOpcode::G_SDIV: + case TargetOpcode::G_UDIV: + case TargetOpcode::G_SREM: + case TargetOpcode::G_UREM: + case TargetOpcode::G_SMIN: + case TargetOpcode::G_SMAX: + case TargetOpcode::G_UMIN: + case TargetOpcode::G_UMAX: + return true; + default: + return false; + } + }; +}; + +// Represents a floating point binary operation. +class GFBinOp : public GBinOp { +public: + static bool classof(const MachineInstr *MI) { + switch (MI->getOpcode()) { + case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMAXNUM: + case TargetOpcode::G_FMINNUM_IEEE: + case TargetOpcode::G_FMAXNUM_IEEE: + case TargetOpcode::G_FMINIMUM: + case TargetOpcode::G_FMAXIMUM: + case TargetOpcode::G_FADD: + case TargetOpcode::G_FSUB: + case TargetOpcode::G_FMUL: + case TargetOpcode::G_FDIV: + case TargetOpcode::G_FPOW: + return true; + default: + return false; + } + }; +}; + +// Represents a logical binary operation. +class GLogicalBinOp : public GBinOp { +public: + static bool classof(const MachineInstr *MI) { + switch (MI->getOpcode()) { + case TargetOpcode::G_AND: + case TargetOpcode::G_OR: + case TargetOpcode::G_XOR: + return true; + default: + return false; + } + }; +}; } // namespace llvm diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 8b15bdb0aca3..5d8def4cca66 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -6390,8 +6390,7 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (TrueValue.isZero() && FalseValue.isOne()) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Inner = MRI.createGenericVirtualRegister(CondTy); - B.buildNot(Inner, Cond); + auto Inner = B.buildNot(CondTy, Cond); B.buildZExtOrTrunc(Dest, Inner); }; return true; @@ -6401,8 +6400,7 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (TrueValue.isZero() && FalseValue.isAllOnes()) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Inner = MRI.createGenericVirtualRegister(CondTy); - B.buildNot(Inner, Cond); + auto Inner = B.buildNot(CondTy, Cond); B.buildSExtOrTrunc(Dest, Inner); }; return true; @@ -6412,8 +6410,7 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (TrueValue - 1 == FalseValue) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Inner = MRI.createGenericVirtualRegister(TrueTy); - B.buildZExtOrTrunc(Inner, Cond); + auto Inner = B.buildZExtOrTrunc(TrueTy, Cond); B.buildAdd(Dest, Inner, False); }; return true; @@ -6423,8 +6420,7 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (TrueValue + 1 == FalseValue) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Inner = MRI.createGenericVirtualRegister(TrueTy); - B.buildSExtOrTrunc(Inner, Cond); + auto Inner = B.buildSExtOrTrunc(TrueTy, Cond); B.buildAdd(Dest, Inner, False); }; return true; @@ -6434,8 +6430,7 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (TrueValue.isPowerOf2() && FalseValue.isZero()) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Inner = MRI.createGenericVirtualRegister(TrueTy); - B.buildZExtOrTrunc(Inner, Cond); + auto Inner = B.buildZExtOrTrunc(TrueTy, Cond); // The shift amount must be scalar. LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy; auto ShAmtC = B.buildConstant(ShiftTy, TrueValue.exactLogBase2()); @@ -6447,8 +6442,7 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (TrueValue.isAllOnes()) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Inner = MRI.createGenericVirtualRegister(TrueTy); - B.buildSExtOrTrunc(Inner, Cond); + auto Inner = B.buildSExtOrTrunc(TrueTy, Cond); B.buildOr(Dest, Inner, False, Flags); }; return true; @@ -6458,10 +6452,8 @@ bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select, if (FalseValue.isAllOnes()) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Not = MRI.createGenericVirtualRegister(CondTy); - B.buildNot(Not, Cond); - Register Inner = MRI.createGenericVirtualRegister(TrueTy); - B.buildSExtOrTrunc(Inner, Not); + auto Not = B.buildNot(CondTy, Cond); + auto Inner = B.buildSExtOrTrunc(TrueTy, Not); B.buildOr(Dest, Inner, True, Flags); }; return true; @@ -6496,8 +6488,7 @@ bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select, if ((Cond == True) || isOneOrOneSplat(True, /* AllowUndefs */ true)) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Ext = MRI.createGenericVirtualRegister(TrueTy); - B.buildZExtOrTrunc(Ext, Cond); + auto Ext = B.buildZExtOrTrunc(TrueTy, Cond); B.buildOr(DstReg, Ext, False, Flags); }; return true; @@ -6508,8 +6499,7 @@ bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select, if ((Cond == False) || isZeroOrZeroSplat(False, /* AllowUndefs */ true)) { MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); - Register Ext = MRI.createGenericVirtualRegister(TrueTy); - B.buildZExtOrTrunc(Ext, Cond); + auto Ext = B.buildZExtOrTrunc(TrueTy, Cond); B.buildAnd(DstReg, Ext, True); }; return true; @@ -6520,11 +6510,9 @@ bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select, MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); // First the not. - Register Inner = MRI.createGenericVirtualRegister(CondTy); - B.buildNot(Inner, Cond); + auto Inner = B.buildNot(CondTy, Cond); // Then an ext to match the destination register. - Register Ext = MRI.createGenericVirtualRegister(TrueTy); - B.buildZExtOrTrunc(Ext, Inner); + auto Ext = B.buildZExtOrTrunc(TrueTy, Inner); B.buildOr(DstReg, Ext, True, Flags); }; return true; @@ -6535,11 +6523,9 @@ bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select, MatchInfo = [=](MachineIRBuilder &B) { B.setInstrAndDebugLoc(*Select); // First the not. - Register Inner = MRI.createGenericVirtualRegister(CondTy); - B.buildNot(Inner, Cond); + auto Inner = B.buildNot(CondTy, Cond); // Then an ext to match the destination register. - Register Ext = MRI.createGenericVirtualRegister(TrueTy); - B.buildZExtOrTrunc(Ext, Inner); + auto Ext = B.buildZExtOrTrunc(TrueTy, Inner); B.buildAnd(DstReg, Ext, False); }; return true; @@ -6548,6 +6534,54 @@ bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select, return false; } +bool CombinerHelper::tryFoldSelectOfBinOps(GSelect *Select, + BuildFnTy &MatchInfo) { + Register DstReg = Select->getReg(0); + Register Cond = Select->getCondReg(); + Register False = Select->getFalseReg(); + Register True = Select->getTrueReg(); + LLT DstTy = MRI.getType(DstReg); + + GBinOp *LHS = getOpcodeDef(True, MRI); + GBinOp *RHS = getOpcodeDef(False, MRI); + + // We need two binops of the same kind on the true/false registers. + if (!LHS || !RHS || LHS->getOpcode() != RHS->getOpcode()) + return false; + + // Note that there are no constraints on CondTy. + unsigned Flags = (LHS->getFlags() & RHS->getFlags()) | Select->getFlags(); + unsigned Opcode = LHS->getOpcode(); + + // Fold select(cond, binop(x, y), binop(z, y)) + // --> binop(select(cond, x, z), y) + if (LHS->getRHSReg() == RHS->getRHSReg()) { + MatchInfo = [=](MachineIRBuilder &B) { + B.setInstrAndDebugLoc(*Select); + auto Sel = B.buildSelect(DstTy, Cond, LHS->getLHSReg(), RHS->getLHSReg(), + Select->getFlags()); + B.buildInstr(Opcode, {DstReg}, {Sel, LHS->getRHSReg()}, Flags); + }; + return true; + } + + // Fold select(cond, binop(x, y), binop(x, z)) + // --> binop(x, select(cond, y, z)) + if (LHS->getLHSReg() == RHS->getLHSReg()) { + MatchInfo = [=](MachineIRBuilder &B) { + B.setInstrAndDebugLoc(*Select); + auto Sel = B.buildSelect(DstTy, Cond, LHS->getRHSReg(), RHS->getRHSReg(), + Select->getFlags()); + B.buildInstr(Opcode, {DstReg}, {LHS->getLHSReg(), Sel}, Flags); + }; + return true; + } + + // FIXME: use isCommutable(). + + return false; +} + bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) { GSelect *Select = cast(&MI); @@ -6557,5 +6591,8 @@ bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) { if (tryFoldBoolSelectToLogic(Select, MatchInfo)) return true; + if (tryFoldSelectOfBinOps(Select, MatchInfo)) + return true; + return false; } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir index be2de620fa45..c5a3490221b6 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir @@ -544,3 +544,154 @@ body: | %ext:_(s32) = G_ANYEXT %sel $w0 = COPY %ext(s32) ... +--- +# select cond, and(x, y), and(z, y) --> and (select cond, x, z), y +name: select_cond_and_x_y_and_z_y_and_select_x_z_y +body: | + bb.1: + liveins: $x0, $x1, $x2 + ; CHECK-LABEL: name: select_cond_and_x_y_and_z_y_and_select_x_z_y + ; CHECK: liveins: $x0, $x1, $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3 + ; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64) + ; CHECK-NEXT: %a:_(s8) = G_TRUNC [[COPY1]](s64) + ; CHECK-NEXT: %b:_(s8) = G_TRUNC [[COPY2]](s64) + ; CHECK-NEXT: %d:_(s8) = G_TRUNC [[COPY3]](s64) + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s8) = G_SELECT %c(s1), %a, %d + ; CHECK-NEXT: %sel:_(s8) = G_AND [[SELECT]], %b + ; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s8) + ; CHECK-NEXT: $w0 = COPY %ext(s32) + %0:_(s64) = COPY $x0 + %1:_(s64) = COPY $x1 + %2:_(s64) = COPY $x2 + %3:_(s64) = COPY $x3 + %4:_(s64) = COPY $x4 + %c:_(s1) = G_TRUNC %0 + %a:_(s8) = G_TRUNC %1 + %b:_(s8) = G_TRUNC %2 + %d:_(s8) = G_TRUNC %3 + %e:_(s8) = G_TRUNC %4 + %and1:_(s8) = G_AND %a, %b + %and2:_(s8) = G_AND %d, %b + %sel:_(s8) = G_SELECT %c, %and1, %and2 + %ext:_(s32) = G_ANYEXT %sel + $w0 = COPY %ext(s32) +... +--- +# select cond, xor(x, y), xor(x, z) --> xor x, select, x, z) +name: select_cond_xor_x_y_xor_x_z_xor_x__select_x_y +body: | + bb.1: + liveins: $x0, $x1, $x2 + ; CHECK-LABEL: name: select_cond_xor_x_y_xor_x_z_xor_x__select_x_y + ; CHECK: liveins: $x0, $x1, $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x4 + ; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64) + ; CHECK-NEXT: %a:_(s8) = G_TRUNC [[COPY1]](s64) + ; CHECK-NEXT: %d:_(s8) = G_TRUNC [[COPY2]](s64) + ; CHECK-NEXT: %e:_(s8) = G_TRUNC [[COPY3]](s64) + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s8) = G_SELECT %c(s1), %e, %d + ; CHECK-NEXT: %sel:_(s8) = G_XOR %a, [[SELECT]] + ; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s8) + ; CHECK-NEXT: $w0 = COPY %ext(s32) + %0:_(s64) = COPY $x0 + %1:_(s64) = COPY $x1 + %2:_(s64) = COPY $x2 + %3:_(s64) = COPY $x3 + %4:_(s64) = COPY $x4 + %c:_(s1) = G_TRUNC %0 + %a:_(s8) = G_TRUNC %1 + %b:_(s8) = G_TRUNC %2 + %d:_(s8) = G_TRUNC %3 + %e:_(s8) = G_TRUNC %4 + %xor1:_(s8) = G_XOR %a, %e + %xor2:_(s8) = G_XOR %a, %d + %sel:_(s8) = G_SELECT %c, %xor1, %xor2 + %ext:_(s32) = G_ANYEXT %sel + $w0 = COPY %ext(s32) +... +--- +# negative test select cond, and(x, y), or(z, a) --> failed +name: select_cond_and_x_y_or_z_a_failed +body: | + bb.1: + liveins: $x0, $x1, $x2 + ; CHECK-LABEL: name: select_cond_and_x_y_or_z_a_failed + ; CHECK: liveins: $x0, $x1, $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4 + ; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64) + ; CHECK-NEXT: %a:_(s8) = G_TRUNC [[COPY1]](s64) + ; CHECK-NEXT: %b:_(s8) = G_TRUNC [[COPY2]](s64) + ; CHECK-NEXT: %d:_(s8) = G_TRUNC [[COPY3]](s64) + ; CHECK-NEXT: %e:_(s8) = G_TRUNC [[COPY4]](s64) + ; CHECK-NEXT: %and1:_(s8) = G_AND %a, %b + ; CHECK-NEXT: %or2:_(s8) = G_OR %e, %d + ; CHECK-NEXT: %sel:_(s8) = G_SELECT %c(s1), %and1, %or2 + ; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s8) + ; CHECK-NEXT: $w0 = COPY %ext(s32) + %0:_(s64) = COPY $x0 + %1:_(s64) = COPY $x1 + %2:_(s64) = COPY $x2 + %3:_(s64) = COPY $x3 + %4:_(s64) = COPY $x4 + %c:_(s1) = G_TRUNC %0 + %a:_(s8) = G_TRUNC %1 + %b:_(s8) = G_TRUNC %2 + %d:_(s8) = G_TRUNC %3 + %e:_(s8) = G_TRUNC %4 + %and1:_(s8) = G_AND %a, %b + %or2:_(s8) = G_OR %e, %d + %sel:_(s8) = G_SELECT %c, %and1, %or2 + %ext:_(s32) = G_ANYEXT %sel + $w0 = COPY %ext(s32) +... +--- +# flags test select cond, xor(x, y), xor(x, z) --> xor x, select, cond, x, z) +name: flags_select_cond_xor_x_y_xor_x_z_xor_x__select_cond_x_y +body: | + bb.1: + liveins: $x0, $x1, $x2 + ; CHECK-LABEL: name: flags_select_cond_xor_x_y_xor_x_z_xor_x__select_cond_x_y + ; CHECK: liveins: $x0, $x1, $x2 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x4 + ; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64) + ; CHECK-NEXT: %a:_(s8) = G_TRUNC [[COPY1]](s64) + ; CHECK-NEXT: %d:_(s8) = G_TRUNC [[COPY2]](s64) + ; CHECK-NEXT: %e:_(s8) = G_TRUNC [[COPY3]](s64) + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s8) = ninf exact G_SELECT %c(s1), %e, %d + ; CHECK-NEXT: %sel:_(s8) = ninf arcp exact G_XOR %a, [[SELECT]] + ; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s8) + ; CHECK-NEXT: $w0 = COPY %ext(s32) + %0:_(s64) = COPY $x0 + %1:_(s64) = COPY $x1 + %2:_(s64) = COPY $x2 + %3:_(s64) = COPY $x3 + %4:_(s64) = COPY $x4 + %c:_(s1) = G_TRUNC %0 + %a:_(s8) = G_TRUNC %1 + %b:_(s8) = G_TRUNC %2 + %d:_(s8) = G_TRUNC %3 + %e:_(s8) = G_TRUNC %4 + %xor1:_(s8) = nsz arcp nsw G_XOR %a, %e + %xor2:_(s8) = nnan arcp nuw G_XOR %a, %d + %sel:_(s8) = ninf exact G_SELECT %c, %xor1, %xor2 + %ext:_(s32) = G_ANYEXT %sel + $w0 = COPY %ext(s32) +... -- GitLab From 0abf3a93a3088140c0585672c8b852e5db93a302 Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Sat, 6 Jan 2024 18:30:37 +0800 Subject: [PATCH 196/728] [X86][NFC] Use single table for EVEX compression This patch is to address my review comments in #77065 to simplify the implemention of EVEX2Legacy compression. --- llvm/lib/Target/X86/X86CompressEVEX.cpp | 26 +++----------- .../TableGen/X86CompressEVEXTablesEmitter.cpp | 35 ++++++------------- 2 files changed, 15 insertions(+), 46 deletions(-) diff --git a/llvm/lib/Target/X86/X86CompressEVEX.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp index accb98cec19d..fc980c611dc6 100644 --- a/llvm/lib/Target/X86/X86CompressEVEX.cpp +++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp @@ -224,16 +224,7 @@ static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) { return true; } -// For EVEX instructions that can be encoded using VEX encoding -// replace them by the VEX encoding in order to reduce size. -static bool CompressEvexToVexImpl(MachineInstr &MI, const X86Subtarget &ST) { - // VEX format. - // # of bytes: 0,2,3 1 1 0,1 0,1,2,4 0,1 - // [Prefixes] [VEX] OPCODE ModR/M [SIB] [DISP] [IMM] - // - // EVEX format. - // # of bytes: 4 1 1 1 4 / 1 1 - // [Prefixes] EVEX Opcode ModR/M [SIB] [Disp32] / [Disp8*N] [Immediate] +static bool CompressEVEXImpl(MachineInstr &MI, const X86Subtarget &ST) { const MCInstrDesc &Desc = MI.getDesc(); // Check for EVEX instructions only. @@ -251,10 +242,7 @@ static bool CompressEvexToVexImpl(MachineInstr &MI, const X86Subtarget &ST) { if (Desc.TSFlags & X86II::EVEX_L2) return false; - // Use the VEX.L bit to select the 128 or 256-bit table. - ArrayRef Table = - (Desc.TSFlags & X86II::VEX_L) ? ArrayRef(X86EvexToVex256CompressTable) - : ArrayRef(X86EvexToVex128CompressTable); + ArrayRef Table = ArrayRef(X86CompressEVEXTable); unsigned Opc = MI.getOpcode(); const auto *I = llvm::lower_bound(Table, Opc); @@ -278,10 +266,8 @@ bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) { // Make sure the tables are sorted. static std::atomic TableChecked(false); if (!TableChecked.load(std::memory_order_relaxed)) { - assert(llvm::is_sorted(X86EvexToVex128CompressTable) && - "X86EvexToVex128CompressTable is not sorted!"); - assert(llvm::is_sorted(X86EvexToVex256CompressTable) && - "X86EvexToVex256CompressTable is not sorted!"); + assert(llvm::is_sorted(X86CompressEVEXTable) && + "X86CompressEVEXTable is not sorted!"); TableChecked.store(true, std::memory_order_relaxed); } #endif @@ -291,12 +277,10 @@ bool CompressEVEXPass::runOnMachineFunction(MachineFunction &MF) { bool Changed = false; - /// Go over all basic blocks in function and replace - /// EVEX encoded instrs by VEX encoding when possible. for (MachineBasicBlock &MBB : MF) { // Traverse the basic block. for (MachineInstr &MI : MBB) - Changed |= CompressEvexToVexImpl(MI, ST); + Changed |= CompressEVEXImpl(MI, ST); } return Changed; diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index a45e87af4a3f..0fcd0b0ab4ff 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -47,9 +47,7 @@ class X86CompressEVEXTablesEmitter { typedef std::pair Entry; - // Represent both compress tables - std::vector EVEX2VEX128; - std::vector EVEX2VEX256; + std::vector Table; public: X86CompressEVEXTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} @@ -64,20 +62,13 @@ private: void X86CompressEVEXTablesEmitter::printTable(const std::vector &Table, raw_ostream &OS) { - StringRef Size = (Table == EVEX2VEX128) ? "128" : "256"; - OS << "// X86 EVEX encoded instructions that have a VEX " << Size - << " encoding\n" - << "// (table format: ).\n" - << "static const X86CompressEVEXTableEntry X86EvexToVex" << Size - << "CompressTable[] = {\n" - << " // EVEX scalar with corresponding VEX.\n"; + OS << "static const X86CompressEVEXTableEntry X86CompressEVEXTable[] = { \n"; // Print all entries added to the table - for (const auto &Pair : Table) { + for (const auto &Pair : Table) OS << " { X86::" << Pair.first->TheDef->getName() << ", X86::" << Pair.second->TheDef->getName() << " },\n"; - } OS << "};\n\n"; } @@ -175,33 +166,27 @@ void X86CompressEVEXTablesEmitter::run(raw_ostream &OS) { const Record *Rec = Inst->TheDef; uint64_t Opcode = getValueFromBitsInit(Inst->TheDef->getValueAsBitsInit("Opcode")); - const CodeGenInstruction *VEXInst = nullptr; + const CodeGenInstruction *NewInst = nullptr; if (ManualMap.find(Rec->getName()) != ManualMap.end()) { Record *NewRec = Records.getDef(ManualMap.at(Rec->getName())); assert(NewRec && "Instruction not found!"); - VEXInst = &Target.getInstruction(NewRec); + NewInst = &Target.getInstruction(NewRec); } else { - // For each EVEX instruction look for a VEX match in the appropriate + // For each pre-compression instruction look for a match in the appropriate // vector (instructions with the same opcode) using function object // IsMatch. auto Match = llvm::find_if(CompressedInsts[Opcode], IsMatch(Inst)); if (Match != CompressedInsts[Opcode].end()) - VEXInst = *Match; + NewInst = *Match; } - if (!VEXInst) + if (!NewInst) continue; - // In case a match is found add new entry to the appropriate table - if (Rec->getValueAsBit("hasVEX_L")) - EVEX2VEX256.push_back(std::make_pair(Inst, VEXInst)); // {0,1} - else - EVEX2VEX128.push_back(std::make_pair(Inst, VEXInst)); // {0,0} + Table.push_back(std::make_pair(Inst, NewInst)); } - // Print both tables - printTable(EVEX2VEX128, OS); - printTable(EVEX2VEX256, OS); + printTable(Table, OS); } } // namespace -- GitLab From bd0dc357af453f03770c5d43c66ee5a3584abdca Mon Sep 17 00:00:00 2001 From: Abhinav271828 <71174780+Abhinav271828@users.noreply.github.com> Date: Sat, 6 Jan 2024 17:08:25 +0530 Subject: [PATCH 197/728] [MLIR][Presburger] Shift GeneratingFunction.h to includes (#77114) We shift the GeneratingFunction.h header file to the include/ directory and wrap it in a `detail` namespace. --- .../Analysis/Presburger/GeneratingFunction.h | 5 ++- .../Analysis/Presburger/CMakeLists.txt | 1 + .../Presburger/GeneratingFunctionTest.cpp | 39 +++++++++++++++++++ mlir/unittests/Analysis/Presburger/Utils.h | 36 ++++++++++++++++- 4 files changed, 79 insertions(+), 2 deletions(-) rename mlir/{lib => include/mlir}/Analysis/Presburger/GeneratingFunction.h (96%) create mode 100644 mlir/unittests/Analysis/Presburger/GeneratingFunctionTest.cpp diff --git a/mlir/lib/Analysis/Presburger/GeneratingFunction.h b/mlir/include/mlir/Analysis/Presburger/GeneratingFunction.h similarity index 96% rename from mlir/lib/Analysis/Presburger/GeneratingFunction.h rename to mlir/include/mlir/Analysis/Presburger/GeneratingFunction.h index f7deba921ea5..4dd692c25156 100644 --- a/mlir/lib/Analysis/Presburger/GeneratingFunction.h +++ b/mlir/include/mlir/Analysis/Presburger/GeneratingFunction.h @@ -19,6 +19,7 @@ namespace mlir { namespace presburger { +namespace detail { // A parametric point is a vector, each of whose elements // is an affine function of n parameters. Each row @@ -83,7 +84,8 @@ public: std::vector> sumDenominators = denominators; sumDenominators.insert(sumDenominators.end(), gf.denominators.begin(), gf.denominators.end()); - return GeneratingFunction(0, sumSigns, sumNumerators, sumDenominators); + return GeneratingFunction(numParam, sumSigns, sumNumerators, + sumDenominators); } llvm::raw_ostream &print(llvm::raw_ostream &os) const { @@ -128,6 +130,7 @@ private: std::vector> denominators; }; +} // namespace detail } // namespace presburger } // namespace mlir diff --git a/mlir/unittests/Analysis/Presburger/CMakeLists.txt b/mlir/unittests/Analysis/Presburger/CMakeLists.txt index e37133354e53..54a841726cd1 100644 --- a/mlir/unittests/Analysis/Presburger/CMakeLists.txt +++ b/mlir/unittests/Analysis/Presburger/CMakeLists.txt @@ -1,5 +1,6 @@ add_mlir_unittest(MLIRPresburgerTests FractionTest.cpp + GeneratingFunctionTest.cpp IntegerPolyhedronTest.cpp IntegerRelationTest.cpp LinearTransformTest.cpp diff --git a/mlir/unittests/Analysis/Presburger/GeneratingFunctionTest.cpp b/mlir/unittests/Analysis/Presburger/GeneratingFunctionTest.cpp new file mode 100644 index 000000000000..5df1a5a0f64c --- /dev/null +++ b/mlir/unittests/Analysis/Presburger/GeneratingFunctionTest.cpp @@ -0,0 +1,39 @@ +//===- MatrixTest.cpp - Tests for QuasiPolynomial -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Analysis/Presburger/GeneratingFunction.h" +#include "./Utils.h" +#include +#include + +using namespace mlir; +using namespace presburger; +using namespace mlir::presburger::detail; + +TEST(GeneratingFunctionTest, sum) { + GeneratingFunction gf1(2, {1, -1}, + {makeFracMatrix(2, 3, {{1, 2, 5}, {7, 2, 6}}), + makeFracMatrix(2, 3, {{5, 2, 5}, {3, 7, 2}})}, + {{{3, 6}, {7, 2}}, {{2, 8}, {6, 3}}}); + GeneratingFunction gf2(2, {1, 1}, + {makeFracMatrix(2, 3, {{6, 2, 1}, {4, 2, 6}}), + makeFracMatrix(2, 3, {{3, 2, 6}, {9, 2, 5}})}, + {{{3, 7}, {5, 1}}, {{5, 2}, {6, 2}}}); + + GeneratingFunction sum = gf1 + gf2; + EXPECT_EQ_REPR_GENERATINGFUNCTION( + sum, GeneratingFunction(2, {1, -1, 1, 1}, + {makeFracMatrix(2, 3, {{1, 2, 5}, {7, 2, 6}}), + makeFracMatrix(2, 3, {{5, 2, 5}, {3, 7, 2}}), + makeFracMatrix(2, 3, {{6, 2, 1}, {4, 2, 6}}), + makeFracMatrix(2, 3, {{3, 2, 6}, {9, 2, 5}})}, + {{{3, 6}, {7, 2}}, + {{2, 8}, {6, 3}}, + {{3, 7}, {5, 1}}, + {{5, 2}, {6, 2}}})); +} diff --git a/mlir/unittests/Analysis/Presburger/Utils.h b/mlir/unittests/Analysis/Presburger/Utils.h index 2a9966c7ce2e..6b00898a7e27 100644 --- a/mlir/unittests/Analysis/Presburger/Utils.h +++ b/mlir/unittests/Analysis/Presburger/Utils.h @@ -13,6 +13,7 @@ #ifndef MLIR_UNITTESTS_ANALYSIS_PRESBURGER_UTILS_H #define MLIR_UNITTESTS_ANALYSIS_PRESBURGER_UTILS_H +#include "mlir/Analysis/Presburger/GeneratingFunction.h" #include "mlir/Analysis/Presburger/IntegerRelation.h" #include "mlir/Analysis/Presburger/Matrix.h" #include "mlir/Analysis/Presburger/PWMAFunction.h" @@ -72,9 +73,42 @@ inline void EXPECT_EQ_FRAC_MATRIX(FracMatrix a, FracMatrix b) { EXPECT_EQ(a(row, col), b(row, col)); } +// Check the coefficients (in order) of two generating functions. +// Note that this is not a true equality check. +inline void EXPECT_EQ_REPR_GENERATINGFUNCTION(detail::GeneratingFunction a, + detail::GeneratingFunction b) { + EXPECT_EQ(a.getNumParams(), b.getNumParams()); + + SmallVector aSigns = a.getSigns(); + SmallVector bSigns = b.getSigns(); + EXPECT_EQ(aSigns.size(), bSigns.size()); + for (unsigned i = 0, e = aSigns.size(); i < e; i++) + EXPECT_EQ(aSigns[i], bSigns[i]); + + std::vector aNums = a.getNumerators(); + std::vector bNums = b.getNumerators(); + EXPECT_EQ(aNums.size(), bNums.size()); + for (unsigned i = 0, e = aNums.size(); i < e; i++) + EXPECT_EQ_FRAC_MATRIX(aNums[i], bNums[i]); + + std::vector> aDens = a.getDenominators(); + std::vector> bDens = b.getDenominators(); + EXPECT_EQ(aDens.size(), bDens.size()); + for (unsigned i = 0, e = aDens.size(); i < e; i++) { + EXPECT_EQ(aDens[i].size(), bDens[i].size()); + for (unsigned j = 0, f = aDens[i].size(); j < f; j++) { + EXPECT_EQ(aDens[i][j].size(), bDens[i][j].size()); + for (unsigned k = 0, g = aDens[i][j].size(); k < g; k++) { + EXPECT_EQ(aDens[i][j][k], bDens[i][j][k]); + } + } + } +} + // Check the coefficients (in order) of two quasipolynomials. // Note that this is not a true equality check. -inline void EXPECT_EQ_REPR_QUASIPOLYNOMIAL(QuasiPolynomial a, QuasiPolynomial b) { +inline void EXPECT_EQ_REPR_QUASIPOLYNOMIAL(QuasiPolynomial a, + QuasiPolynomial b) { EXPECT_EQ(a.getNumInputs(), b.getNumInputs()); SmallVector aCoeffs = a.getCoefficients(), -- GitLab From d08482924efe8b2c44913583af7b8f60a29975d1 Mon Sep 17 00:00:00 2001 From: Qizhi Hu <836744285@qq.com> Date: Sat, 6 Jan 2024 19:50:00 +0800 Subject: [PATCH 198/728] [clang-tidy] fix false positive in cppcoreguidelines-missing-std-forward (#77056) Parameter variable which is forwarded in lambda capture list or in body by reference is reasonable and current version of this check produces false positive on these cases. This patch try to fix the [issue](https://github.com/llvm/llvm-project/issues/68105) Co-authored-by: huqizhi <836744285@qq.com> --- .../MissingStdForwardCheck.cpp | 60 ++++++++++++++++++- clang-tools-extra/docs/ReleaseNotes.rst | 4 ++ .../cppcoreguidelines/missing-std-forward.cpp | 31 +++++++++- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp index 0b85ea19735e..370de12999ac 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp @@ -53,16 +53,72 @@ AST_MATCHER(ParmVarDecl, isTemplateTypeParameter) { FuncTemplate->getTemplateParameters()->getDepth(); } +AST_MATCHER_P(NamedDecl, hasSameNameAsBoundNode, std::string, BindingID) { + IdentifierInfo *II = Node.getIdentifier(); + if (nullptr == II) + return false; + StringRef Name = II->getName(); + + return Builder->removeBindings( + [this, Name](const ast_matchers::internal::BoundNodesMap &Nodes) { + const DynTypedNode &BN = Nodes.getNode(this->BindingID); + if (const auto *ND = BN.get()) { + if (!isa(ND)) + return true; + return ND->getName() != Name; + } + return true; + }); +} + +AST_MATCHER_P(LambdaCapture, hasCaptureKind, LambdaCaptureKind, Kind) { + return Node.getCaptureKind() == Kind; +} + +AST_MATCHER_P(LambdaExpr, hasCaptureDefaultKind, LambdaCaptureDefault, Kind) { + return Node.getCaptureDefault() == Kind; +} + } // namespace void MissingStdForwardCheck::registerMatchers(MatchFinder *Finder) { + auto RefToParmImplicit = allOf( + equalsBoundNode("var"), hasInitializer(ignoringParenImpCasts( + declRefExpr(to(equalsBoundNode("param")))))); + auto RefToParm = capturesVar( + varDecl(anyOf(hasSameNameAsBoundNode("param"), RefToParmImplicit))); + auto HasRefToParm = hasAnyCapture(RefToParm); + + auto CaptureInRef = + allOf(hasCaptureDefaultKind(LambdaCaptureDefault::LCD_ByRef), + unless(hasAnyCapture( + capturesVar(varDecl(hasSameNameAsBoundNode("param")))))); + auto CaptureInCopy = allOf( + hasCaptureDefaultKind(LambdaCaptureDefault::LCD_ByCopy), HasRefToParm); + auto CaptureByRefExplicit = hasAnyCapture( + allOf(hasCaptureKind(LambdaCaptureKind::LCK_ByRef), RefToParm)); + + auto CapturedInBody = + lambdaExpr(anyOf(CaptureInRef, CaptureInCopy, CaptureByRefExplicit)); + auto CapturedInCaptureList = hasAnyCapture(capturesVar( + varDecl(hasInitializer(ignoringParenImpCasts(equalsBoundNode("call")))))); + + auto CapturedInLambda = hasDeclContext(cxxRecordDecl( + isLambda(), + hasParent(lambdaExpr(forCallable(equalsBoundNode("func")), + anyOf(CapturedInCaptureList, CapturedInBody))))); + auto ToParam = hasAnyParameter(parmVarDecl(equalsBoundNode("param"))); auto ForwardCallMatcher = callExpr( - forCallable(equalsBoundNode("func")), argumentCountIs(1), + callExpr().bind("call"), argumentCountIs(1), + hasArgument( + 0, declRefExpr(to( + varDecl(optionally(equalsBoundNode("param"))).bind("var")))), + forCallable(anyOf(equalsBoundNode("func"), CapturedInLambda)), callee(unresolvedLookupExpr(hasAnyDeclaration( namedDecl(hasUnderlyingDecl(hasName("::std::forward")))))), - hasArgument(0, declRefExpr(to(equalsBoundNode("param"))).bind("ref")), + unless(anyOf(hasAncestor(typeLoc()), hasAncestor(expr(hasUnevaluatedContext()))))); diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 08ade306b5a0..1bd5a72126c1 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -301,6 +301,10 @@ Changes in existing checks coroutine functions and increase issue detection for cases involving type aliases with references. +- Improved :doc:`cppcoreguidelines-missing-std-forward + ` check to + address false positives in the capture list and body of lambdas. + - Improved :doc:`cppcoreguidelines-narrowing-conversions ` check by extending the `IgnoreConversionFromTypes` option to include types without a diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp index b9720db272e4..443f338ba204 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp @@ -90,9 +90,9 @@ void lambda_value_capture(T&& t) { } template -void lambda_value_reference(T&& t) { - // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: forwarding reference parameter 't' is never forwarded inside the function body [cppcoreguidelines-missing-std-forward] - [&]() { T other = std::forward(t); }; +void lambda_value_capture_copy(T&& t) { + // CHECK-MESSAGES: :[[@LINE-1]]:36: warning: forwarding reference parameter 't' is never forwarded inside the function body [cppcoreguidelines-missing-std-forward] + [&,t]() { T other = std::forward(t); }; } } // namespace positive_cases @@ -147,4 +147,29 @@ class AClass { T data; }; +template +void lambda_value_reference(T&& t) { + [&]() { T other = std::forward(t); }; +} + +template +void lambda_value_reference_capture_list_ref_1(T&& t) { + [=, &t] { T other = std::forward(t); }; +} + +template +void lambda_value_reference_capture_list_ref_2(T&& t) { + [&t] { T other = std::forward(t); }; +} + +template +void lambda_value_reference_capture_list(T&& t) { + [t = std::forward(t)] { t(); }; +} + +template +void lambda_value_reference_auxiliary_var(T&& t) { + [&x = t]() { T other = std::forward(x); }; +} + } // namespace negative_cases -- GitLab From 3fb0d8dc808cb7f315670d76109edc9c57cb3d90 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 6 Jan 2024 12:08:03 +0000 Subject: [PATCH 199/728] Recommit "[VPlan] Mark Select VPInstructions as not having sideeffects." With #70253 landed, selects for reduction results are explicitly used by ComputeReductionResult and Selects can be marked as not having side-effects again. This reverts the revert commit 173032902c960d4d0d67b521d8c149553d8e8ba3. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 + llvm/test/Transforms/LoopVectorize/reduction-small-size.ll | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1e5273bcd748..c9def41a3cec 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -121,6 +121,7 @@ bool VPRecipeBase::mayHaveSideEffects() const { case VPInstructionSC: switch (cast(this)->getOpcode()) { case Instruction::ICmp: + case Instruction::Select: case VPInstruction::Not: case VPInstruction::CalculateTripCountMinusVF: case VPInstruction::CanonicalIVIncrementForPart: diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll index 3973a288fe71..2a58748d8fb6 100644 --- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll +++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll @@ -11,15 +11,12 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) { ; CHECK: vector.ph: ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> undef, <4 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI]], ; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT2]] ; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8> @@ -40,7 +37,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) { ; CHECK: for.body: ; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] ; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[R_NEXT:%.*]], [[IF_END]] ] -; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END]] ; CHECK: if.then: ; CHECK-NEXT: [[T0:%.*]] = sdiv i32 undef, undef ; CHECK-NEXT: br label [[IF_END]] -- GitLab From 4b9bbd38686af3dbffd45b360bd5af629426bdbc Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Sat, 6 Jan 2024 21:38:12 +0800 Subject: [PATCH 200/728] [X86][NFC] Refine code in X86CompressEVEXTablesEmitter.cpp 1. Simplify getValueFromBitsInit about cast and return type 2. Remove out-of-date comments and allow memory ops in function object `IsMatch` so that we can reuse it for EVEX2Legacy compression. This patch is to extract NFC in #77065 into a separate commit. --- .../TableGen/X86CompressEVEXTablesEmitter.cpp | 54 ++++++++----------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp index 0fcd0b0ab4ff..82a9bfee0115 100644 --- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp @@ -73,15 +73,14 @@ void X86CompressEVEXTablesEmitter::printTable(const std::vector &Table, OS << "};\n\n"; } -// Return true if the 2 BitsInits are equal -// Calculates the integer value residing BitsInit object -static inline uint64_t getValueFromBitsInit(const BitsInit *B) { - uint64_t Value = 0; - for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { - if (BitInit *Bit = dyn_cast(B->getBit(i))) - Value |= uint64_t(Bit->getValue()) << i; - else - PrintFatalError("Invalid VectSize bit"); +static uint8_t byteFromBitsInit(const BitsInit *B) { + unsigned N = B->getNumBits(); + assert(N <= 8 && "Field is too large for uint8_t!"); + + uint8_t Value = 0; + for (unsigned I = 0; I != N; ++I) { + BitInit *Bit = cast(B->getBit(I)); + Value |= Bit->getValue() << I; } return Value; } @@ -105,30 +104,23 @@ public: NewRI.Form)) return false; - // This is needed for instructions with intrinsic version (_Int). - // Where the only difference is the size of the operands. - // For example: VUCOMISDZrm and Int_VUCOMISDrm - // Also for instructions that their EVEX version was upgraded to work with - // k-registers. For example VPCMPEQBrm (xmm output register) and - // VPCMPEQBZ128rm (k register output register). - for (unsigned i = 0, e = OldInst->Operands.size(); i < e; i++) { - Record *OpRec1 = OldInst->Operands[i].Rec; - Record *OpRec2 = NewInst->Operands[i].Rec; - - if (OpRec1 == OpRec2) + for (unsigned I = 0, E = OldInst->Operands.size(); I < E; ++I) { + Record *OldOpRec = OldInst->Operands[I].Rec; + Record *NewOpRec = NewInst->Operands[I].Rec; + + if (OldOpRec == NewOpRec) continue; - if (isRegisterOperand(OpRec1) && isRegisterOperand(OpRec2)) { - if (getRegOperandSize(OpRec1) != getRegOperandSize(OpRec2)) + if (isRegisterOperand(OldOpRec) && isRegisterOperand(NewOpRec)) { + if (getRegOperandSize(OldOpRec) != getRegOperandSize(NewOpRec)) + return false; + } else if (isMemoryOperand(OldOpRec) && isMemoryOperand(NewOpRec)) { + if (getMemOperandSize(OldOpRec) != getMemOperandSize(NewOpRec)) return false; - } else if (isMemoryOperand(OpRec1) && isMemoryOperand(OpRec2)) { - return false; - } else if (isImmediateOperand(OpRec1) && isImmediateOperand(OpRec2)) { - if (OpRec1->getValueAsDef("Type") != OpRec2->getValueAsDef("Type")) { + } else if (isImmediateOperand(OldOpRec) && isImmediateOperand(NewOpRec)) { + if (OldOpRec->getValueAsDef("Type") != NewOpRec->getValueAsDef("Type")) return false; - } - } else - return false; + } } return true; @@ -164,8 +156,8 @@ void X86CompressEVEXTablesEmitter::run(raw_ostream &OS) { for (const CodeGenInstruction *Inst : PreCompressionInsts) { const Record *Rec = Inst->TheDef; - uint64_t Opcode = - getValueFromBitsInit(Inst->TheDef->getValueAsBitsInit("Opcode")); + uint8_t Opcode = + byteFromBitsInit(Inst->TheDef->getValueAsBitsInit("Opcode")); const CodeGenInstruction *NewInst = nullptr; if (ManualMap.find(Rec->getName()) != ManualMap.end()) { Record *NewRec = Records.getDef(ManualMap.at(Rec->getName())); -- GitLab From 8bbf100799a97f8342bf1a8409c6fb48f03e837f Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Sat, 6 Jan 2024 22:03:03 +0800 Subject: [PATCH 201/728] [X86][NFC] Remove dead code for "_REV" instructions Those "_REV" instructions should not appear before encoding optimization, while macro fusion and flag-copy lowering are before encoding optimization. --- llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h | 16 ---------------- llvm/lib/Target/X86/X86FlagsCopyLowering.cpp | 1 - 2 files changed, 17 deletions(-) diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h index e006dd877360..304b998e1f26 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h +++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h @@ -148,25 +148,21 @@ classifyFirstOpcodeInMacroFusion(unsigned Opcode) { case X86::AND16ri8: case X86::AND16rm: case X86::AND16rr: - case X86::AND16rr_REV: case X86::AND32i32: case X86::AND32ri: case X86::AND32ri8: case X86::AND32rm: case X86::AND32rr: - case X86::AND32rr_REV: case X86::AND64i32: case X86::AND64ri32: case X86::AND64ri8: case X86::AND64rm: case X86::AND64rr: - case X86::AND64rr_REV: case X86::AND8i8: case X86::AND8ri: case X86::AND8ri8: case X86::AND8rm: case X86::AND8rr: - case X86::AND8rr_REV: return FirstMacroFusionInstKind::And; // CMP case X86::CMP16i16: @@ -175,28 +171,24 @@ classifyFirstOpcodeInMacroFusion(unsigned Opcode) { case X86::CMP16ri8: case X86::CMP16rm: case X86::CMP16rr: - case X86::CMP16rr_REV: case X86::CMP32i32: case X86::CMP32mr: case X86::CMP32ri: case X86::CMP32ri8: case X86::CMP32rm: case X86::CMP32rr: - case X86::CMP32rr_REV: case X86::CMP64i32: case X86::CMP64mr: case X86::CMP64ri32: case X86::CMP64ri8: case X86::CMP64rm: case X86::CMP64rr: - case X86::CMP64rr_REV: case X86::CMP8i8: case X86::CMP8mr: case X86::CMP8ri: case X86::CMP8ri8: case X86::CMP8rm: case X86::CMP8rr: - case X86::CMP8rr_REV: return FirstMacroFusionInstKind::Cmp; // ADD case X86::ADD16i16: @@ -204,50 +196,42 @@ classifyFirstOpcodeInMacroFusion(unsigned Opcode) { case X86::ADD16ri8: case X86::ADD16rm: case X86::ADD16rr: - case X86::ADD16rr_REV: case X86::ADD32i32: case X86::ADD32ri: case X86::ADD32ri8: case X86::ADD32rm: case X86::ADD32rr: - case X86::ADD32rr_REV: case X86::ADD64i32: case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD64rm: case X86::ADD64rr: - case X86::ADD64rr_REV: case X86::ADD8i8: case X86::ADD8ri: case X86::ADD8ri8: case X86::ADD8rm: case X86::ADD8rr: - case X86::ADD8rr_REV: // SUB case X86::SUB16i16: case X86::SUB16ri: case X86::SUB16ri8: case X86::SUB16rm: case X86::SUB16rr: - case X86::SUB16rr_REV: case X86::SUB32i32: case X86::SUB32ri: case X86::SUB32ri8: case X86::SUB32rm: case X86::SUB32rr: - case X86::SUB32rr_REV: case X86::SUB64i32: case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB64rm: case X86::SUB64rr: - case X86::SUB64rr_REV: case X86::SUB8i8: case X86::SUB8ri: case X86::SUB8ri8: case X86::SUB8rm: case X86::SUB8rr: - case X86::SUB8rr_REV: return FirstMacroFusionInstKind::AddSub; // INC case X86::INC16r: diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp index b13bf361ab79..aad839b83ee1 100644 --- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp +++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp @@ -173,7 +173,6 @@ static FlagArithMnemonic getMnemonicFromOpcode(unsigned Opcode) { #define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC) \ LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr) \ - LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr_REV) \ LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rm) \ LLVM_EXPAND_INSTR_SIZES(MNEMONIC, mr) \ case X86::MNEMONIC##8ri: \ -- GitLab From 0c7d46a7fd5b7956e285d385a6945153d6a06eb0 Mon Sep 17 00:00:00 2001 From: Younan Zhang Date: Sat, 6 Jan 2024 22:26:34 +0800 Subject: [PATCH 202/728] [Clang] Correctly construct template arguments for template template parameters (#76811) This fixes the bug introduced by https://github.com/llvm/llvm-project/commit/6db007a0654ed7a6ed5c3aa3b61a937c19a6bc6b. We construct placeholder template arguments for template-template parameters to avoid mismatching argument substitution since they have different depths with their corresponding template arguments. In this case, ```cpp template